1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPURegisterBankInfo.h" 17 #include "GCNSubtarget.h" 18 #include "MCTargetDesc/AMDGPUInstPrinter.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/MachineDominators.h" 23 #include "llvm/CodeGen/RegisterScavenging.h" 24 25 using namespace llvm; 26 27 #define GET_REGINFO_TARGET_DESC 28 #include "AMDGPUGenRegisterInfo.inc" 29 30 static cl::opt<bool> EnableSpillSGPRToVGPR( 31 "amdgpu-spill-sgpr-to-vgpr", 32 cl::desc("Enable spilling VGPRs to SGPRs"), 33 cl::ReallyHidden, 34 cl::init(true)); 35 36 std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts; 37 std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable; 38 39 // Map numbers of DWORDs to indexes in SubRegFromChannelTable. 40 // Valid indexes are shifted 1, such that a 0 mapping means unsupported. 41 // e.g. for 8 DWORDs (256-bit), SubRegFromChannelTableWidthMap[8] = 8, 42 // meaning index 7 in SubRegFromChannelTable. 43 static const std::array<unsigned, 17> SubRegFromChannelTableWidthMap = { 44 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9}; 45 46 namespace llvm { 47 48 // A temporary struct to spill SGPRs. 49 // This is mostly to spill SGPRs to memory. Spilling SGPRs into VGPR lanes emits 50 // just v_writelane and v_readlane. 51 // 52 // When spilling to memory, the SGPRs are written into VGPR lanes and the VGPR 53 // is saved to scratch (or the other way around for loads). 54 // For this, a VGPR is required where the needed lanes can be clobbered. The 55 // RegScavenger can provide a VGPR where currently active lanes can be 56 // clobbered, but we still need to save inactive lanes. 57 // The high-level steps are: 58 // - Try to scavenge SGPR(s) to save exec 59 // - Try to scavenge VGPR 60 // - Save needed, all or inactive lanes of a TmpVGPR 61 // - Spill/Restore SGPRs using TmpVGPR 62 // - Restore TmpVGPR 63 // 64 // To save all lanes of TmpVGPR, exec needs to be saved and modified. If we 65 // cannot scavenge temporary SGPRs to save exec, we use the following code: 66 // buffer_store_dword TmpVGPR ; only if active lanes need to be saved 67 // s_not exec, exec 68 // buffer_store_dword TmpVGPR ; save inactive lanes 69 // s_not exec, exec 70 struct SGPRSpillBuilder { 71 struct PerVGPRData { 72 unsigned PerVGPR; 73 unsigned NumVGPRs; 74 int64_t VGPRLanes; 75 }; 76 77 // The SGPR to save 78 Register SuperReg; 79 MachineBasicBlock::iterator MI; 80 ArrayRef<int16_t> SplitParts; 81 unsigned NumSubRegs; 82 bool IsKill; 83 const DebugLoc &DL; 84 85 /* When spilling to stack */ 86 // The SGPRs are written into this VGPR, which is then written to scratch 87 // (or vice versa for loads). 88 Register TmpVGPR = AMDGPU::NoRegister; 89 // Temporary spill slot to save TmpVGPR to. 90 int TmpVGPRIndex = 0; 91 // If TmpVGPR is live before the spill or if it is scavenged. 92 bool TmpVGPRLive = false; 93 // Scavenged SGPR to save EXEC. 94 Register SavedExecReg = AMDGPU::NoRegister; 95 // Stack index to write the SGPRs to. 96 int Index; 97 unsigned EltSize = 4; 98 99 RegScavenger *RS; 100 MachineBasicBlock *MBB; 101 MachineFunction &MF; 102 SIMachineFunctionInfo &MFI; 103 const SIInstrInfo &TII; 104 const SIRegisterInfo &TRI; 105 bool IsWave32; 106 Register ExecReg; 107 unsigned MovOpc; 108 unsigned NotOpc; 109 110 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 111 bool IsWave32, MachineBasicBlock::iterator MI, int Index, 112 RegScavenger *RS) 113 : SGPRSpillBuilder(TRI, TII, IsWave32, MI, MI->getOperand(0).getReg(), 114 MI->getOperand(0).isKill(), Index, RS) {} 115 116 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 117 bool IsWave32, MachineBasicBlock::iterator MI, Register Reg, 118 bool IsKill, int Index, RegScavenger *RS) 119 : SuperReg(Reg), MI(MI), IsKill(IsKill), DL(MI->getDebugLoc()), 120 Index(Index), RS(RS), MBB(MI->getParent()), MF(*MBB->getParent()), 121 MFI(*MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI), 122 IsWave32(IsWave32) { 123 const TargetRegisterClass *RC = TRI.getPhysRegClass(SuperReg); 124 SplitParts = TRI.getRegSplitParts(RC, EltSize); 125 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 126 127 if (IsWave32) { 128 ExecReg = AMDGPU::EXEC_LO; 129 MovOpc = AMDGPU::S_MOV_B32; 130 NotOpc = AMDGPU::S_NOT_B32; 131 } else { 132 ExecReg = AMDGPU::EXEC; 133 MovOpc = AMDGPU::S_MOV_B64; 134 NotOpc = AMDGPU::S_NOT_B64; 135 } 136 137 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 138 assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI && 139 SuperReg != AMDGPU::EXEC && "exec should never spill"); 140 } 141 142 PerVGPRData getPerVGPRData() { 143 PerVGPRData Data; 144 Data.PerVGPR = IsWave32 ? 32 : 64; 145 Data.NumVGPRs = (NumSubRegs + (Data.PerVGPR - 1)) / Data.PerVGPR; 146 Data.VGPRLanes = (1LL << std::min(Data.PerVGPR, NumSubRegs)) - 1LL; 147 return Data; 148 } 149 150 // Tries to scavenge SGPRs to save EXEC and a VGPR. Uses v0 if no VGPR is 151 // free. 152 // Writes these instructions if an SGPR can be scavenged: 153 // s_mov_b64 s[6:7], exec ; Save exec 154 // s_mov_b64 exec, 3 ; Wanted lanemask 155 // buffer_store_dword v1 ; Write scavenged VGPR to emergency slot 156 // 157 // Writes these instructions if no SGPR can be scavenged: 158 // buffer_store_dword v0 ; Only if no free VGPR was found 159 // s_not_b64 exec, exec 160 // buffer_store_dword v0 ; Save inactive lanes 161 // ; exec stays inverted, it is flipped back in 162 // ; restore. 163 void prepare() { 164 // Scavenged temporary VGPR to use. It must be scavenged once for any number 165 // of spilled subregs. 166 // FIXME: The liveness analysis is limited and does not tell if a register 167 // is in use in lanes that are currently inactive. We can never be sure if 168 // a register as actually in use in another lane, so we need to save all 169 // used lanes of the chosen VGPR. 170 assert(RS && "Cannot spill SGPR to memory without RegScavenger"); 171 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0, false); 172 173 // Reserve temporary stack slot 174 TmpVGPRIndex = MFI.getScavengeFI(MF.getFrameInfo(), TRI); 175 if (TmpVGPR) { 176 // Found a register that is dead in the currently active lanes, we only 177 // need to spill inactive lanes. 178 TmpVGPRLive = false; 179 } else { 180 // Pick v0 because it doesn't make a difference. 181 TmpVGPR = AMDGPU::VGPR0; 182 TmpVGPRLive = true; 183 } 184 185 if (TmpVGPRLive) { 186 // We need to inform the scavenger that this index is already in use until 187 // we're done with the custom emergency spill. 188 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR); 189 } 190 191 // We may end up recursively calling the scavenger, and don't want to re-use 192 // the same register. 193 RS->setRegUsed(TmpVGPR); 194 195 // Try to scavenge SGPRs to save exec 196 assert(!SavedExecReg && "Exec is already saved, refuse to save again"); 197 const TargetRegisterClass &RC = 198 IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass; 199 RS->setRegUsed(SuperReg); 200 SavedExecReg = RS->scavengeRegister(&RC, MI, 0, false); 201 202 int64_t VGPRLanes = getPerVGPRData().VGPRLanes; 203 204 if (SavedExecReg) { 205 RS->setRegUsed(SavedExecReg); 206 // Set exec to needed lanes 207 BuildMI(*MBB, MI, DL, TII.get(MovOpc), SavedExecReg).addReg(ExecReg); 208 auto I = 209 BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg).addImm(VGPRLanes); 210 if (!TmpVGPRLive) 211 I.addReg(TmpVGPR, RegState::ImplicitDefine); 212 // Spill needed lanes 213 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 214 } else { 215 // The modify and restore of exec clobber SCC, which we would have to save 216 // and restore. FIXME: We probably would need to reserve a register for 217 // this. 218 if (RS->isRegUsed(AMDGPU::SCC)) 219 MI->emitError("unhandled SGPR spill to memory"); 220 221 // Spill active lanes 222 if (TmpVGPRLive) 223 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false, 224 /*IsKill*/ false); 225 // Spill inactive lanes 226 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 227 if (!TmpVGPRLive) 228 I.addReg(TmpVGPR, RegState::ImplicitDefine); 229 I->getOperand(2).setIsDead(true); // Mark SCC as dead. 230 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 231 } 232 } 233 234 // Writes these instructions if an SGPR can be scavenged: 235 // buffer_load_dword v1 ; Write scavenged VGPR to emergency slot 236 // s_waitcnt vmcnt(0) ; If a free VGPR was found 237 // s_mov_b64 exec, s[6:7] ; Save exec 238 // 239 // Writes these instructions if no SGPR can be scavenged: 240 // buffer_load_dword v0 ; Restore inactive lanes 241 // s_waitcnt vmcnt(0) ; If a free VGPR was found 242 // s_not_b64 exec, exec 243 // buffer_load_dword v0 ; Only if no free VGPR was found 244 void restore() { 245 if (SavedExecReg) { 246 // Restore used lanes 247 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 248 /*IsKill*/ false); 249 // Restore exec 250 auto I = BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg) 251 .addReg(SavedExecReg, RegState::Kill); 252 // Add an implicit use of the load so it is not dead. 253 // FIXME This inserts an unnecessary waitcnt 254 if (!TmpVGPRLive) { 255 I.addReg(TmpVGPR, RegState::ImplicitKill); 256 } 257 } else { 258 // Restore inactive lanes 259 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 260 /*IsKill*/ false); 261 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 262 if (!TmpVGPRLive) 263 I.addReg(TmpVGPR, RegState::ImplicitKill); 264 I->getOperand(2).setIsDead(true); // Mark SCC as dead. 265 266 // Restore active lanes 267 if (TmpVGPRLive) 268 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true); 269 } 270 271 // Inform the scavenger where we're releasing our custom scavenged register. 272 if (TmpVGPRLive) { 273 MachineBasicBlock::iterator RestorePt = std::prev(MI); 274 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR, &*RestorePt); 275 } 276 } 277 278 // Write TmpVGPR to memory or read TmpVGPR from memory. 279 // Either using a single buffer_load/store if exec is set to the needed mask 280 // or using 281 // buffer_load 282 // s_not exec, exec 283 // buffer_load 284 // s_not exec, exec 285 void readWriteTmpVGPR(unsigned Offset, bool IsLoad) { 286 if (SavedExecReg) { 287 // Spill needed lanes 288 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 289 } else { 290 // The modify and restore of exec clobber SCC, which we would have to save 291 // and restore. FIXME: We probably would need to reserve a register for 292 // this. 293 if (RS->isRegUsed(AMDGPU::SCC)) 294 MI->emitError("unhandled SGPR spill to memory"); 295 296 // Spill active lanes 297 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad, 298 /*IsKill*/ false); 299 // Spill inactive lanes 300 auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 301 Not0->getOperand(2).setIsDead(); // Mark SCC as dead. 302 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 303 auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 304 Not1->getOperand(2).setIsDead(); // Mark SCC as dead. 305 } 306 } 307 308 void setMI(MachineBasicBlock *NewMBB, MachineBasicBlock::iterator NewMI) { 309 assert(MBB->getParent() == &MF); 310 MI = NewMI; 311 MBB = NewMBB; 312 } 313 }; 314 315 } // namespace llvm 316 317 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 318 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 319 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 320 321 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 322 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 323 (getSubRegIndexLaneMask(AMDGPU::lo16) | 324 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 325 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 326 "getNumCoveredRegs() will not work with generated subreg masks!"); 327 328 RegPressureIgnoredUnits.resize(getNumRegUnits()); 329 RegPressureIgnoredUnits.set( 330 *MCRegUnitIterator(MCRegister::from(AMDGPU::M0), this)); 331 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 332 RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this)); 333 334 // HACK: Until this is fully tablegen'd. 335 static llvm::once_flag InitializeRegSplitPartsFlag; 336 337 static auto InitializeRegSplitPartsOnce = [this]() { 338 for (unsigned Idx = 1, E = getNumSubRegIndices() - 1; Idx < E; ++Idx) { 339 unsigned Size = getSubRegIdxSize(Idx); 340 if (Size & 31) 341 continue; 342 std::vector<int16_t> &Vec = RegSplitParts[Size / 32 - 1]; 343 unsigned Pos = getSubRegIdxOffset(Idx); 344 if (Pos % Size) 345 continue; 346 Pos /= Size; 347 if (Vec.empty()) { 348 unsigned MaxNumParts = 1024 / Size; // Maximum register is 1024 bits. 349 Vec.resize(MaxNumParts); 350 } 351 Vec[Pos] = Idx; 352 } 353 }; 354 355 static llvm::once_flag InitializeSubRegFromChannelTableFlag; 356 357 static auto InitializeSubRegFromChannelTableOnce = [this]() { 358 for (auto &Row : SubRegFromChannelTable) 359 Row.fill(AMDGPU::NoSubRegister); 360 for (uint16_t Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { 361 unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; 362 unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; 363 assert(Width < SubRegFromChannelTableWidthMap.size()); 364 Width = SubRegFromChannelTableWidthMap[Width]; 365 if (Width == 0) 366 continue; 367 unsigned TableIdx = Width - 1; 368 assert(TableIdx < SubRegFromChannelTable.size()); 369 assert(Offset < SubRegFromChannelTable[TableIdx].size()); 370 SubRegFromChannelTable[TableIdx][Offset] = Idx; 371 } 372 }; 373 374 llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce); 375 llvm::call_once(InitializeSubRegFromChannelTableFlag, 376 InitializeSubRegFromChannelTableOnce); 377 } 378 379 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 380 MCRegister Reg) const { 381 MCRegAliasIterator R(Reg, this, true); 382 383 for (; R.isValid(); ++R) 384 Reserved.set(*R); 385 } 386 387 // Forced to be here by one .inc 388 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 389 const MachineFunction *MF) const { 390 CallingConv::ID CC = MF->getFunction().getCallingConv(); 391 switch (CC) { 392 case CallingConv::C: 393 case CallingConv::Fast: 394 case CallingConv::Cold: 395 return MF->getSubtarget<GCNSubtarget>().hasGFX90AInsts() 396 ? CSR_AMDGPU_HighRegs_With_AGPRs_SaveList 397 : CSR_AMDGPU_HighRegs_SaveList; 398 case CallingConv::AMDGPU_Gfx: 399 return MF->getSubtarget<GCNSubtarget>().hasGFX90AInsts() 400 ? CSR_AMDGPU_SI_Gfx_With_AGPRs_SaveList 401 : CSR_AMDGPU_SI_Gfx_SaveList; 402 default: { 403 // Dummy to not crash RegisterClassInfo. 404 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 405 return &NoCalleeSavedReg; 406 } 407 } 408 } 409 410 const MCPhysReg * 411 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 412 return nullptr; 413 } 414 415 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 416 CallingConv::ID CC) const { 417 switch (CC) { 418 case CallingConv::C: 419 case CallingConv::Fast: 420 case CallingConv::Cold: 421 return MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts() 422 ? CSR_AMDGPU_HighRegs_With_AGPRs_RegMask 423 : CSR_AMDGPU_HighRegs_RegMask; 424 case CallingConv::AMDGPU_Gfx: 425 return MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts() 426 ? CSR_AMDGPU_SI_Gfx_With_AGPRs_RegMask 427 : CSR_AMDGPU_SI_Gfx_RegMask; 428 default: 429 return nullptr; 430 } 431 } 432 433 const uint32_t *SIRegisterInfo::getNoPreservedMask() const { 434 return CSR_AMDGPU_NoRegs_RegMask; 435 } 436 437 const TargetRegisterClass * 438 SIRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, 439 const MachineFunction &MF) const { 440 // FIXME: Should have a helper function like getEquivalentVGPRClass to get the 441 // equivalent AV class. If used one, the verifier will crash after 442 // RegBankSelect in the GISel flow. The aligned regclasses are not fully given 443 // until Instruction selection. 444 if (MF.getSubtarget<GCNSubtarget>().hasMAIInsts() && 445 (isVGPRClass(RC) || isAGPRClass(RC))) { 446 if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass) 447 return &AMDGPU::AV_32RegClass; 448 if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass) 449 return &AMDGPU::AV_64RegClass; 450 if (RC == &AMDGPU::VReg_64_Align2RegClass || 451 RC == &AMDGPU::AReg_64_Align2RegClass) 452 return &AMDGPU::AV_64_Align2RegClass; 453 if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass) 454 return &AMDGPU::AV_96RegClass; 455 if (RC == &AMDGPU::VReg_96_Align2RegClass || 456 RC == &AMDGPU::AReg_96_Align2RegClass) 457 return &AMDGPU::AV_96_Align2RegClass; 458 if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass) 459 return &AMDGPU::AV_128RegClass; 460 if (RC == &AMDGPU::VReg_128_Align2RegClass || 461 RC == &AMDGPU::AReg_128_Align2RegClass) 462 return &AMDGPU::AV_128_Align2RegClass; 463 if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass) 464 return &AMDGPU::AV_160RegClass; 465 if (RC == &AMDGPU::VReg_160_Align2RegClass || 466 RC == &AMDGPU::AReg_160_Align2RegClass) 467 return &AMDGPU::AV_160_Align2RegClass; 468 if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass) 469 return &AMDGPU::AV_192RegClass; 470 if (RC == &AMDGPU::VReg_192_Align2RegClass || 471 RC == &AMDGPU::AReg_192_Align2RegClass) 472 return &AMDGPU::AV_192_Align2RegClass; 473 if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass) 474 return &AMDGPU::AV_256RegClass; 475 if (RC == &AMDGPU::VReg_256_Align2RegClass || 476 RC == &AMDGPU::AReg_256_Align2RegClass) 477 return &AMDGPU::AV_256_Align2RegClass; 478 if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass) 479 return &AMDGPU::AV_512RegClass; 480 if (RC == &AMDGPU::VReg_512_Align2RegClass || 481 RC == &AMDGPU::AReg_512_Align2RegClass) 482 return &AMDGPU::AV_512_Align2RegClass; 483 if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass) 484 return &AMDGPU::AV_1024RegClass; 485 if (RC == &AMDGPU::VReg_1024_Align2RegClass || 486 RC == &AMDGPU::AReg_1024_Align2RegClass) 487 return &AMDGPU::AV_1024_Align2RegClass; 488 } 489 490 return TargetRegisterInfo::getLargestLegalSuperClass(RC, MF); 491 } 492 493 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 494 const SIFrameLowering *TFI = 495 MF.getSubtarget<GCNSubtarget>().getFrameLowering(); 496 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 497 // During ISel lowering we always reserve the stack pointer in entry 498 // functions, but never actually want to reference it when accessing our own 499 // frame. If we need a frame pointer we use it, but otherwise we can just use 500 // an immediate "0" which we represent by returning NoRegister. 501 if (FuncInfo->isEntryFunction()) { 502 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 503 } 504 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 505 : FuncInfo->getStackPtrOffsetReg(); 506 } 507 508 bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const { 509 // When we need stack realignment, we can't reference off of the 510 // stack pointer, so we reserve a base pointer. 511 const MachineFrameInfo &MFI = MF.getFrameInfo(); 512 return MFI.getNumFixedObjects() && shouldRealignStack(MF); 513 } 514 515 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; } 516 517 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 518 return CSR_AMDGPU_AllVGPRs_RegMask; 519 } 520 521 const uint32_t *SIRegisterInfo::getAllAGPRRegMask() const { 522 return CSR_AMDGPU_AllAGPRs_RegMask; 523 } 524 525 const uint32_t *SIRegisterInfo::getAllVectorRegMask() const { 526 return CSR_AMDGPU_AllVectorRegs_RegMask; 527 } 528 529 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 530 return CSR_AMDGPU_AllAllocatableSRegs_RegMask; 531 } 532 533 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 534 unsigned NumRegs) { 535 assert(NumRegs < SubRegFromChannelTableWidthMap.size()); 536 unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs]; 537 assert(NumRegIndex && "Not implemented"); 538 assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].size()); 539 return SubRegFromChannelTable[NumRegIndex - 1][Channel]; 540 } 541 542 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 543 const MachineFunction &MF) const { 544 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 545 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 546 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 547 } 548 549 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 550 BitVector Reserved(getNumRegs()); 551 Reserved.set(AMDGPU::MODE); 552 553 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 554 // this seems likely to result in bugs, so I'm marking them as reserved. 555 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 556 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 557 558 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 559 reserveRegisterTuples(Reserved, AMDGPU::M0); 560 561 // Reserve src_vccz, src_execz, src_scc. 562 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 563 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 564 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 565 566 // Reserve the memory aperture registers. 567 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 568 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 569 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 570 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 571 572 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 573 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 574 575 // Reserve xnack_mask registers - support is not implemented in Codegen. 576 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 577 578 // Reserve lds_direct register - support is not implemented in Codegen. 579 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 580 581 // Reserve Trap Handler registers - support is not implemented in Codegen. 582 reserveRegisterTuples(Reserved, AMDGPU::TBA); 583 reserveRegisterTuples(Reserved, AMDGPU::TMA); 584 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 585 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 586 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 587 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 588 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 589 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 590 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 591 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 592 593 // Reserve null register - it shall never be allocated 594 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL); 595 596 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 597 // will result in bugs. 598 if (isWave32) { 599 Reserved.set(AMDGPU::VCC); 600 Reserved.set(AMDGPU::VCC_HI); 601 } 602 603 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 604 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 605 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 606 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 607 reserveRegisterTuples(Reserved, Reg); 608 } 609 610 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 611 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 612 unsigned MaxNumAGPRs = MaxNumVGPRs; 613 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 614 615 if (ST.hasGFX90AInsts()) { 616 // In an entry function without calls and AGPRs used it is possible to use 617 // the whole register budget for VGPRs. 618 619 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and 620 // split register file accordingly. 621 if (MFI->usesAGPRs(MF)) { 622 MaxNumVGPRs /= 2; 623 MaxNumAGPRs = MaxNumVGPRs; 624 } else { 625 if (MaxNumVGPRs > TotalNumVGPRs) { 626 MaxNumAGPRs = MaxNumVGPRs - TotalNumVGPRs; 627 MaxNumVGPRs = TotalNumVGPRs; 628 } else 629 MaxNumAGPRs = 0; 630 } 631 } else if (ST.hasMAIInsts() && MFI->usesAGPRs(MF)) { 632 // In order to guarantee copying between AGPRs, we need a scratch VGPR 633 // available at all times. 634 reserveRegisterTuples(Reserved, AMDGPU::VGPR32); 635 } 636 637 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 638 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 639 reserveRegisterTuples(Reserved, Reg); 640 } 641 642 for (unsigned i = MaxNumAGPRs; i < TotalNumVGPRs; ++i) { 643 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 644 reserveRegisterTuples(Reserved, Reg); 645 } 646 647 for (auto Reg : AMDGPU::SReg_32RegClass) { 648 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 649 Register Low = getSubReg(Reg, AMDGPU::lo16); 650 // This is to prevent BB vcc liveness errors. 651 if (!AMDGPU::SGPR_LO16RegClass.contains(Low)) 652 Reserved.set(Low); 653 } 654 655 for (auto Reg : AMDGPU::AGPR_32RegClass) { 656 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 657 } 658 659 // Reserve all the rest AGPRs if there are no instructions to use it. 660 if (!ST.hasMAIInsts()) { 661 for (unsigned i = 0; i < MaxNumVGPRs; ++i) { 662 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 663 reserveRegisterTuples(Reserved, Reg); 664 } 665 } 666 667 Register ScratchRSrcReg = MFI->getScratchRSrcReg(); 668 if (ScratchRSrcReg != AMDGPU::NoRegister) { 669 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 670 // to spill. 671 // TODO: May need to reserve a VGPR if doing LDS spilling. 672 reserveRegisterTuples(Reserved, ScratchRSrcReg); 673 } 674 675 // We have to assume the SP is needed in case there are calls in the function, 676 // which is detected after the function is lowered. If we aren't really going 677 // to need SP, don't bother reserving it. 678 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 679 680 if (StackPtrReg) { 681 reserveRegisterTuples(Reserved, StackPtrReg); 682 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 683 } 684 685 MCRegister FrameReg = MFI->getFrameOffsetReg(); 686 if (FrameReg) { 687 reserveRegisterTuples(Reserved, FrameReg); 688 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 689 } 690 691 if (hasBasePointer(MF)) { 692 MCRegister BasePtrReg = getBaseRegister(); 693 reserveRegisterTuples(Reserved, BasePtrReg); 694 assert(!isSubRegister(ScratchRSrcReg, BasePtrReg)); 695 } 696 697 for (auto Reg : MFI->WWMReservedRegs) { 698 reserveRegisterTuples(Reserved, Reg.first); 699 } 700 701 // Reserve VGPRs used for SGPR spilling. 702 // Note we treat freezeReservedRegs unusually because we run register 703 // allocation in two phases. It's OK to re-freeze with new registers for the 704 // second run. 705 #if 0 706 for (auto &SpilledFI : MFI->sgpr_spill_vgprs()) { 707 for (auto &SpilledVGPR : SpilledFI.second) 708 reserveRegisterTuples(Reserved, SpilledVGPR.VGPR); 709 } 710 #endif 711 712 // FIXME: Stop using reserved registers for this. 713 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 714 reserveRegisterTuples(Reserved, Reg); 715 716 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 717 reserveRegisterTuples(Reserved, Reg); 718 719 for (auto SSpill : MFI->getSGPRSpillVGPRs()) 720 reserveRegisterTuples(Reserved, SSpill.VGPR); 721 722 return Reserved; 723 } 724 725 bool SIRegisterInfo::isAsmClobberable(const MachineFunction &MF, 726 MCRegister PhysReg) const { 727 return !MF.getRegInfo().isReserved(PhysReg); 728 } 729 730 bool SIRegisterInfo::shouldRealignStack(const MachineFunction &MF) const { 731 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 732 // On entry, the base address is 0, so it can't possibly need any more 733 // alignment. 734 735 // FIXME: Should be able to specify the entry frame alignment per calling 736 // convention instead. 737 if (Info->isEntryFunction()) 738 return false; 739 740 return TargetRegisterInfo::shouldRealignStack(MF); 741 } 742 743 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 744 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 745 if (Info->isEntryFunction()) { 746 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 747 return MFI.hasStackObjects() || MFI.hasCalls(); 748 } 749 750 // May need scavenger for dealing with callee saved registers. 751 return true; 752 } 753 754 bool SIRegisterInfo::requiresFrameIndexScavenging( 755 const MachineFunction &MF) const { 756 // Do not use frame virtual registers. They used to be used for SGPRs, but 757 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 758 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 759 // spill. 760 return false; 761 } 762 763 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 764 const MachineFunction &MF) const { 765 const MachineFrameInfo &MFI = MF.getFrameInfo(); 766 return MFI.hasStackObjects(); 767 } 768 769 bool SIRegisterInfo::requiresVirtualBaseRegisters( 770 const MachineFunction &) const { 771 // There are no special dedicated stack or frame pointers. 772 return true; 773 } 774 775 int64_t SIRegisterInfo::getScratchInstrOffset(const MachineInstr *MI) const { 776 assert(SIInstrInfo::isMUBUF(*MI) || SIInstrInfo::isFLATScratch(*MI)); 777 778 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 779 AMDGPU::OpName::offset); 780 return MI->getOperand(OffIdx).getImm(); 781 } 782 783 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 784 int Idx) const { 785 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 786 return 0; 787 788 assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 789 AMDGPU::OpName::vaddr) || 790 (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 791 AMDGPU::OpName::saddr))) && 792 "Should never see frame index on non-address operand"); 793 794 return getScratchInstrOffset(MI); 795 } 796 797 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 798 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 799 return false; 800 801 int64_t FullOffset = Offset + getScratchInstrOffset(MI); 802 803 if (SIInstrInfo::isMUBUF(*MI)) 804 return !SIInstrInfo::isLegalMUBUFImmOffset(FullOffset); 805 806 const SIInstrInfo *TII = ST.getInstrInfo(); 807 return !TII->isLegalFLATOffset(FullOffset, AMDGPUAS::PRIVATE_ADDRESS, 808 SIInstrFlags::FlatScratch); 809 } 810 811 Register SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 812 int FrameIdx, 813 int64_t Offset) const { 814 MachineBasicBlock::iterator Ins = MBB->begin(); 815 DebugLoc DL; // Defaults to "unknown" 816 817 if (Ins != MBB->end()) 818 DL = Ins->getDebugLoc(); 819 820 MachineFunction *MF = MBB->getParent(); 821 const SIInstrInfo *TII = ST.getInstrInfo(); 822 MachineRegisterInfo &MRI = MF->getRegInfo(); 823 unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32 824 : AMDGPU::V_MOV_B32_e32; 825 826 Register BaseReg = MRI.createVirtualRegister( 827 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XEXEC_HIRegClass 828 : &AMDGPU::VGPR_32RegClass); 829 830 if (Offset == 0) { 831 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), BaseReg) 832 .addFrameIndex(FrameIdx); 833 return BaseReg; 834 } 835 836 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 837 838 Register FIReg = MRI.createVirtualRegister( 839 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass 840 : &AMDGPU::VGPR_32RegClass); 841 842 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 843 .addImm(Offset); 844 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), FIReg) 845 .addFrameIndex(FrameIdx); 846 847 if (ST.enableFlatScratch() ) { 848 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_I32), BaseReg) 849 .addReg(OffsetReg, RegState::Kill) 850 .addReg(FIReg); 851 return BaseReg; 852 } 853 854 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 855 .addReg(OffsetReg, RegState::Kill) 856 .addReg(FIReg) 857 .addImm(0); // clamp bit 858 859 return BaseReg; 860 } 861 862 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 863 int64_t Offset) const { 864 const SIInstrInfo *TII = ST.getInstrInfo(); 865 bool IsFlat = TII->isFLATScratch(MI); 866 867 #ifndef NDEBUG 868 // FIXME: Is it possible to be storing a frame index to itself? 869 bool SeenFI = false; 870 for (const MachineOperand &MO: MI.operands()) { 871 if (MO.isFI()) { 872 if (SeenFI) 873 llvm_unreachable("should not see multiple frame indices"); 874 875 SeenFI = true; 876 } 877 } 878 #endif 879 880 MachineOperand *FIOp = 881 TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr 882 : AMDGPU::OpName::vaddr); 883 884 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 885 int64_t NewOffset = OffsetOp->getImm() + Offset; 886 887 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 888 assert(TII->isMUBUF(MI) || TII->isFLATScratch(MI)); 889 890 if (IsFlat) { 891 assert(TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 892 SIInstrFlags::FlatScratch) && 893 "offset should be legal"); 894 FIOp->ChangeToRegister(BaseReg, false); 895 OffsetOp->setImm(NewOffset); 896 return; 897 } 898 899 #ifndef NDEBUG 900 MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 901 assert(SOffset->isImm() && SOffset->getImm() == 0); 902 #endif 903 904 assert(SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 905 "offset should be legal"); 906 907 FIOp->ChangeToRegister(BaseReg, false); 908 OffsetOp->setImm(NewOffset); 909 } 910 911 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 912 Register BaseReg, 913 int64_t Offset) const { 914 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 915 return false; 916 917 int64_t NewOffset = Offset + getScratchInstrOffset(MI); 918 919 if (SIInstrInfo::isMUBUF(*MI)) 920 return SIInstrInfo::isLegalMUBUFImmOffset(NewOffset); 921 922 const SIInstrInfo *TII = ST.getInstrInfo(); 923 return TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 924 SIInstrFlags::FlatScratch); 925 } 926 927 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 928 const MachineFunction &MF, unsigned Kind) const { 929 // This is inaccurate. It depends on the instruction and address space. The 930 // only place where we should hit this is for dealing with frame indexes / 931 // private accesses, so this is correct in that case. 932 return &AMDGPU::VGPR_32RegClass; 933 } 934 935 const TargetRegisterClass * 936 SIRegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { 937 if (isAGPRClass(RC) && !ST.hasGFX90AInsts()) 938 return getEquivalentVGPRClass(RC); 939 940 return RC; 941 } 942 943 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 944 945 switch (Op) { 946 case AMDGPU::SI_SPILL_S1024_SAVE: 947 case AMDGPU::SI_SPILL_S1024_RESTORE: 948 case AMDGPU::SI_SPILL_V1024_SAVE: 949 case AMDGPU::SI_SPILL_V1024_RESTORE: 950 case AMDGPU::SI_SPILL_A1024_SAVE: 951 case AMDGPU::SI_SPILL_A1024_RESTORE: 952 case AMDGPU::SI_SPILL_AV1024_SAVE: 953 case AMDGPU::SI_SPILL_AV1024_RESTORE: 954 return 32; 955 case AMDGPU::SI_SPILL_S512_SAVE: 956 case AMDGPU::SI_SPILL_S512_RESTORE: 957 case AMDGPU::SI_SPILL_V512_SAVE: 958 case AMDGPU::SI_SPILL_V512_RESTORE: 959 case AMDGPU::SI_SPILL_A512_SAVE: 960 case AMDGPU::SI_SPILL_A512_RESTORE: 961 case AMDGPU::SI_SPILL_AV512_SAVE: 962 case AMDGPU::SI_SPILL_AV512_RESTORE: 963 return 16; 964 case AMDGPU::SI_SPILL_S256_SAVE: 965 case AMDGPU::SI_SPILL_S256_RESTORE: 966 case AMDGPU::SI_SPILL_V256_SAVE: 967 case AMDGPU::SI_SPILL_V256_RESTORE: 968 case AMDGPU::SI_SPILL_A256_SAVE: 969 case AMDGPU::SI_SPILL_A256_RESTORE: 970 case AMDGPU::SI_SPILL_AV256_SAVE: 971 case AMDGPU::SI_SPILL_AV256_RESTORE: 972 return 8; 973 case AMDGPU::SI_SPILL_S224_SAVE: 974 case AMDGPU::SI_SPILL_S224_RESTORE: 975 case AMDGPU::SI_SPILL_V224_SAVE: 976 case AMDGPU::SI_SPILL_V224_RESTORE: 977 case AMDGPU::SI_SPILL_A224_SAVE: 978 case AMDGPU::SI_SPILL_A224_RESTORE: 979 case AMDGPU::SI_SPILL_AV224_SAVE: 980 case AMDGPU::SI_SPILL_AV224_RESTORE: 981 return 7; 982 case AMDGPU::SI_SPILL_S192_SAVE: 983 case AMDGPU::SI_SPILL_S192_RESTORE: 984 case AMDGPU::SI_SPILL_V192_SAVE: 985 case AMDGPU::SI_SPILL_V192_RESTORE: 986 case AMDGPU::SI_SPILL_A192_SAVE: 987 case AMDGPU::SI_SPILL_A192_RESTORE: 988 case AMDGPU::SI_SPILL_AV192_SAVE: 989 case AMDGPU::SI_SPILL_AV192_RESTORE: 990 return 6; 991 case AMDGPU::SI_SPILL_S160_SAVE: 992 case AMDGPU::SI_SPILL_S160_RESTORE: 993 case AMDGPU::SI_SPILL_V160_SAVE: 994 case AMDGPU::SI_SPILL_V160_RESTORE: 995 case AMDGPU::SI_SPILL_A160_SAVE: 996 case AMDGPU::SI_SPILL_A160_RESTORE: 997 case AMDGPU::SI_SPILL_AV160_SAVE: 998 case AMDGPU::SI_SPILL_AV160_RESTORE: 999 return 5; 1000 case AMDGPU::SI_SPILL_S128_SAVE: 1001 case AMDGPU::SI_SPILL_S128_RESTORE: 1002 case AMDGPU::SI_SPILL_V128_SAVE: 1003 case AMDGPU::SI_SPILL_V128_RESTORE: 1004 case AMDGPU::SI_SPILL_A128_SAVE: 1005 case AMDGPU::SI_SPILL_A128_RESTORE: 1006 case AMDGPU::SI_SPILL_AV128_SAVE: 1007 case AMDGPU::SI_SPILL_AV128_RESTORE: 1008 return 4; 1009 case AMDGPU::SI_SPILL_S96_SAVE: 1010 case AMDGPU::SI_SPILL_S96_RESTORE: 1011 case AMDGPU::SI_SPILL_V96_SAVE: 1012 case AMDGPU::SI_SPILL_V96_RESTORE: 1013 case AMDGPU::SI_SPILL_A96_SAVE: 1014 case AMDGPU::SI_SPILL_A96_RESTORE: 1015 case AMDGPU::SI_SPILL_AV96_SAVE: 1016 case AMDGPU::SI_SPILL_AV96_RESTORE: 1017 return 3; 1018 case AMDGPU::SI_SPILL_S64_SAVE: 1019 case AMDGPU::SI_SPILL_S64_RESTORE: 1020 case AMDGPU::SI_SPILL_V64_SAVE: 1021 case AMDGPU::SI_SPILL_V64_RESTORE: 1022 case AMDGPU::SI_SPILL_A64_SAVE: 1023 case AMDGPU::SI_SPILL_A64_RESTORE: 1024 case AMDGPU::SI_SPILL_AV64_SAVE: 1025 case AMDGPU::SI_SPILL_AV64_RESTORE: 1026 return 2; 1027 case AMDGPU::SI_SPILL_S32_SAVE: 1028 case AMDGPU::SI_SPILL_S32_RESTORE: 1029 case AMDGPU::SI_SPILL_V32_SAVE: 1030 case AMDGPU::SI_SPILL_V32_RESTORE: 1031 case AMDGPU::SI_SPILL_A32_SAVE: 1032 case AMDGPU::SI_SPILL_A32_RESTORE: 1033 case AMDGPU::SI_SPILL_AV32_SAVE: 1034 case AMDGPU::SI_SPILL_AV32_RESTORE: 1035 return 1; 1036 default: llvm_unreachable("Invalid spill opcode"); 1037 } 1038 } 1039 1040 static int getOffsetMUBUFStore(unsigned Opc) { 1041 switch (Opc) { 1042 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 1043 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1044 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 1045 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 1046 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 1047 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 1048 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 1049 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 1050 case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN: 1051 return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET; 1052 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 1053 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 1054 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 1055 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 1056 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 1057 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 1058 default: 1059 return -1; 1060 } 1061 } 1062 1063 static int getOffsetMUBUFLoad(unsigned Opc) { 1064 switch (Opc) { 1065 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 1066 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1067 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 1068 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 1069 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 1070 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 1071 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 1072 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 1073 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 1074 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 1075 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 1076 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 1077 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN: 1078 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET; 1079 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 1080 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 1081 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 1082 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 1083 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 1084 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 1085 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 1086 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 1087 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 1088 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 1089 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 1090 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 1091 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 1092 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 1093 default: 1094 return -1; 1095 } 1096 } 1097 1098 static int getOffenMUBUFStore(unsigned Opc) { 1099 switch (Opc) { 1100 case AMDGPU::BUFFER_STORE_DWORD_OFFSET: 1101 return AMDGPU::BUFFER_STORE_DWORD_OFFEN; 1102 case AMDGPU::BUFFER_STORE_BYTE_OFFSET: 1103 return AMDGPU::BUFFER_STORE_BYTE_OFFEN; 1104 case AMDGPU::BUFFER_STORE_SHORT_OFFSET: 1105 return AMDGPU::BUFFER_STORE_SHORT_OFFEN; 1106 case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET: 1107 return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN; 1108 case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET: 1109 return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN; 1110 case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET: 1111 return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN; 1112 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET: 1113 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN; 1114 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET: 1115 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN; 1116 default: 1117 return -1; 1118 } 1119 } 1120 1121 static int getOffenMUBUFLoad(unsigned Opc) { 1122 switch (Opc) { 1123 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET: 1124 return AMDGPU::BUFFER_LOAD_DWORD_OFFEN; 1125 case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET: 1126 return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN; 1127 case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET: 1128 return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN; 1129 case AMDGPU::BUFFER_LOAD_USHORT_OFFSET: 1130 return AMDGPU::BUFFER_LOAD_USHORT_OFFEN; 1131 case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET: 1132 return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN; 1133 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET: 1134 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN; 1135 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET: 1136 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN; 1137 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET: 1138 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN; 1139 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET: 1140 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN; 1141 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET: 1142 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN; 1143 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET: 1144 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN; 1145 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET: 1146 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN; 1147 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET: 1148 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN; 1149 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET: 1150 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN; 1151 default: 1152 return -1; 1153 } 1154 } 1155 1156 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 1157 MachineBasicBlock &MBB, 1158 MachineBasicBlock::iterator MI, 1159 int Index, unsigned Lane, 1160 unsigned ValueReg, bool IsKill) { 1161 MachineFunction *MF = MBB.getParent(); 1162 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1163 const SIInstrInfo *TII = ST.getInstrInfo(); 1164 1165 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 1166 1167 if (Reg == AMDGPU::NoRegister) 1168 return MachineInstrBuilder(); 1169 1170 bool IsStore = MI->mayStore(); 1171 MachineRegisterInfo &MRI = MF->getRegInfo(); 1172 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 1173 1174 unsigned Dst = IsStore ? Reg : ValueReg; 1175 unsigned Src = IsStore ? ValueReg : Reg; 1176 bool IsVGPR = TRI->isVGPR(MRI, Reg); 1177 DebugLoc DL = MI->getDebugLoc(); 1178 if (IsVGPR == TRI->isVGPR(MRI, ValueReg)) { 1179 // Spiller during regalloc may restore a spilled register to its superclass. 1180 // It could result in AGPR spills restored to VGPRs or the other way around, 1181 // making the src and dst with identical regclasses at this point. It just 1182 // needs a copy in such cases. 1183 auto CopyMIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), Dst) 1184 .addReg(Src, getKillRegState(IsKill)); 1185 CopyMIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1186 return CopyMIB; 1187 } 1188 unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 1189 : AMDGPU::V_ACCVGPR_READ_B32_e64; 1190 1191 auto MIB = BuildMI(MBB, MI, DL, TII->get(Opc), Dst) 1192 .addReg(Src, getKillRegState(IsKill)); 1193 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1194 return MIB; 1195 } 1196 1197 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 1198 // need to handle the case where an SGPR may need to be spilled while spilling. 1199 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 1200 MachineFrameInfo &MFI, 1201 MachineBasicBlock::iterator MI, 1202 int Index, 1203 int64_t Offset) { 1204 const SIInstrInfo *TII = ST.getInstrInfo(); 1205 MachineBasicBlock *MBB = MI->getParent(); 1206 const DebugLoc &DL = MI->getDebugLoc(); 1207 bool IsStore = MI->mayStore(); 1208 1209 unsigned Opc = MI->getOpcode(); 1210 int LoadStoreOp = IsStore ? 1211 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 1212 if (LoadStoreOp == -1) 1213 return false; 1214 1215 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 1216 if (spillVGPRtoAGPR(ST, *MBB, MI, Index, 0, Reg->getReg(), false).getInstr()) 1217 return true; 1218 1219 MachineInstrBuilder NewMI = 1220 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 1221 .add(*Reg) 1222 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 1223 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 1224 .addImm(Offset) 1225 .addImm(0) // cpol 1226 .addImm(0) // tfe 1227 .addImm(0) // swz 1228 .cloneMemRefs(*MI); 1229 1230 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 1231 AMDGPU::OpName::vdata_in); 1232 if (VDataIn) 1233 NewMI.add(*VDataIn); 1234 return true; 1235 } 1236 1237 static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII, 1238 unsigned LoadStoreOp, 1239 unsigned EltSize) { 1240 bool IsStore = TII->get(LoadStoreOp).mayStore(); 1241 bool HasVAddr = AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) != -1; 1242 bool UseST = 1243 !HasVAddr && 1244 AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::saddr) < 0; 1245 1246 switch (EltSize) { 1247 case 4: 1248 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1249 : AMDGPU::SCRATCH_LOAD_DWORD_SADDR; 1250 break; 1251 case 8: 1252 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR 1253 : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR; 1254 break; 1255 case 12: 1256 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR 1257 : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR; 1258 break; 1259 case 16: 1260 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR 1261 : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR; 1262 break; 1263 default: 1264 llvm_unreachable("Unexpected spill load/store size!"); 1265 } 1266 1267 if (HasVAddr) 1268 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1269 else if (UseST) 1270 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1271 1272 return LoadStoreOp; 1273 } 1274 1275 void SIRegisterInfo::buildSpillLoadStore( 1276 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, 1277 unsigned LoadStoreOp, int Index, Register ValueReg, bool IsKill, 1278 MCRegister ScratchOffsetReg, int64_t InstOffset, MachineMemOperand *MMO, 1279 RegScavenger *RS, LivePhysRegs *LiveRegs) const { 1280 assert((!RS || !LiveRegs) && "Only RS or LiveRegs can be set but not both"); 1281 1282 MachineFunction *MF = MBB.getParent(); 1283 const SIInstrInfo *TII = ST.getInstrInfo(); 1284 const MachineFrameInfo &MFI = MF->getFrameInfo(); 1285 const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>(); 1286 1287 const MCInstrDesc *Desc = &TII->get(LoadStoreOp); 1288 bool IsStore = Desc->mayStore(); 1289 bool IsFlat = TII->isFLATScratch(LoadStoreOp); 1290 1291 bool CanClobberSCC = false; 1292 bool Scavenged = false; 1293 MCRegister SOffset = ScratchOffsetReg; 1294 1295 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 1296 // On gfx90a+ AGPR is a regular VGPR acceptable for loads and stores. 1297 const bool IsAGPR = !ST.hasGFX90AInsts() && isAGPRClass(RC); 1298 const unsigned RegWidth = AMDGPU::getRegBitWidth(RC->getID()) / 8; 1299 1300 // Always use 4 byte operations for AGPRs because we need to scavenge 1301 // a temporary VGPR. 1302 unsigned EltSize = (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u) : 4u; 1303 unsigned NumSubRegs = RegWidth / EltSize; 1304 unsigned Size = NumSubRegs * EltSize; 1305 unsigned RemSize = RegWidth - Size; 1306 unsigned NumRemSubRegs = RemSize ? 1 : 0; 1307 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 1308 int64_t MaterializedOffset = Offset; 1309 1310 int64_t MaxOffset = Offset + Size + RemSize - EltSize; 1311 int64_t ScratchOffsetRegDelta = 0; 1312 1313 if (IsFlat && EltSize > 4) { 1314 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1315 Desc = &TII->get(LoadStoreOp); 1316 } 1317 1318 Align Alignment = MFI.getObjectAlign(Index); 1319 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 1320 1321 assert((IsFlat || ((Offset % EltSize) == 0)) && 1322 "unexpected VGPR spill offset"); 1323 1324 // Track a VGPR to use for a constant offset we need to materialize. 1325 Register TmpOffsetVGPR; 1326 1327 // Track a VGPR to use as an intermediate value. 1328 Register TmpIntermediateVGPR; 1329 bool UseVGPROffset = false; 1330 1331 // Materialize a VGPR offset required for the given SGPR/VGPR/Immediate 1332 // combination. 1333 auto MaterializeVOffset = [&](Register SGPRBase, Register TmpVGPR, 1334 int64_t VOffset) { 1335 // We are using a VGPR offset 1336 if (IsFlat && SGPRBase) { 1337 // We only have 1 VGPR offset, or 1 SGPR offset. We don't have a free 1338 // SGPR, so perform the add as vector. 1339 // We don't need a base SGPR in the kernel. 1340 1341 if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) >= 2) { 1342 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), TmpVGPR) 1343 .addReg(SGPRBase) 1344 .addImm(VOffset) 1345 .addImm(0); // clamp 1346 } else { 1347 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1348 .addReg(SGPRBase); 1349 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), TmpVGPR) 1350 .addImm(VOffset) 1351 .addReg(TmpOffsetVGPR); 1352 } 1353 } else { 1354 assert(TmpOffsetVGPR); 1355 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1356 .addImm(VOffset); 1357 } 1358 }; 1359 1360 bool IsOffsetLegal = 1361 IsFlat ? TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS, 1362 SIInstrFlags::FlatScratch) 1363 : SIInstrInfo::isLegalMUBUFImmOffset(MaxOffset); 1364 if (!IsOffsetLegal || (IsFlat && !SOffset && !ST.hasFlatScratchSTMode())) { 1365 SOffset = MCRegister(); 1366 1367 // We don't have access to the register scavenger if this function is called 1368 // during PEI::scavengeFrameVirtualRegs() so use LiveRegs in this case. 1369 // TODO: Clobbering SCC is not necessary for scratch instructions in the 1370 // entry. 1371 if (RS) { 1372 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 1373 1374 // Piggy back on the liveness scan we just did see if SCC is dead. 1375 CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC); 1376 } else if (LiveRegs) { 1377 CanClobberSCC = !LiveRegs->contains(AMDGPU::SCC); 1378 for (MCRegister Reg : AMDGPU::SGPR_32RegClass) { 1379 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1380 SOffset = Reg; 1381 break; 1382 } 1383 } 1384 } 1385 1386 if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC) 1387 SOffset = Register(); 1388 1389 if (!SOffset) { 1390 UseVGPROffset = true; 1391 1392 if (RS) { 1393 TmpOffsetVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1394 } else { 1395 assert(LiveRegs); 1396 for (MCRegister Reg : AMDGPU::VGPR_32RegClass) { 1397 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1398 TmpOffsetVGPR = Reg; 1399 break; 1400 } 1401 } 1402 } 1403 1404 assert(TmpOffsetVGPR); 1405 } else if (!SOffset && CanClobberSCC) { 1406 // There are no free SGPRs, and since we are in the process of spilling 1407 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 1408 // on SI/CI and on VI it is true until we implement spilling using scalar 1409 // stores), we have no way to free up an SGPR. Our solution here is to 1410 // add the offset directly to the ScratchOffset or StackPtrOffset 1411 // register, and then subtract the offset after the spill to return the 1412 // register to it's original value. 1413 1414 // TODO: If we don't have to do an emergency stack slot spill, converting 1415 // to use the VGPR offset is fewer instructions. 1416 if (!ScratchOffsetReg) 1417 ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg(); 1418 SOffset = ScratchOffsetReg; 1419 ScratchOffsetRegDelta = Offset; 1420 } else { 1421 Scavenged = true; 1422 } 1423 1424 // We currently only support spilling VGPRs to EltSize boundaries, meaning 1425 // we can simplify the adjustment of Offset here to just scale with 1426 // WavefrontSize. 1427 if (!IsFlat && !UseVGPROffset) 1428 Offset *= ST.getWavefrontSize(); 1429 1430 if (!UseVGPROffset && !SOffset) 1431 report_fatal_error("could not scavenge SGPR to spill in entry function"); 1432 1433 if (UseVGPROffset) { 1434 // We are using a VGPR offset 1435 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, Offset); 1436 } else if (ScratchOffsetReg == AMDGPU::NoRegister) { 1437 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset); 1438 } else { 1439 assert(Offset != 0); 1440 auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1441 .addReg(ScratchOffsetReg) 1442 .addImm(Offset); 1443 Add->getOperand(3).setIsDead(); // Mark SCC as dead. 1444 } 1445 1446 Offset = 0; 1447 } 1448 1449 if (IsFlat && SOffset == AMDGPU::NoRegister) { 1450 assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0 1451 && "Unexpected vaddr for flat scratch with a FI operand"); 1452 1453 if (UseVGPROffset) { 1454 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1455 } else { 1456 assert(ST.hasFlatScratchSTMode()); 1457 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1458 } 1459 1460 Desc = &TII->get(LoadStoreOp); 1461 } 1462 1463 for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e; 1464 ++i, RegOffset += EltSize) { 1465 if (i == NumSubRegs) { 1466 EltSize = RemSize; 1467 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1468 } 1469 Desc = &TII->get(LoadStoreOp); 1470 1471 if (!IsFlat && UseVGPROffset) { 1472 int NewLoadStoreOp = IsStore ? getOffenMUBUFStore(LoadStoreOp) 1473 : getOffenMUBUFLoad(LoadStoreOp); 1474 Desc = &TII->get(NewLoadStoreOp); 1475 } 1476 1477 if (UseVGPROffset && TmpOffsetVGPR == TmpIntermediateVGPR) { 1478 // If we are spilling an AGPR beyond the range of the memory instruction 1479 // offset and need to use a VGPR offset, we ideally have at least 2 1480 // scratch VGPRs. If we don't have a second free VGPR without spilling, 1481 // recycle the VGPR used for the offset which requires resetting after 1482 // each subregister. 1483 1484 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, MaterializedOffset); 1485 } 1486 1487 unsigned NumRegs = EltSize / 4; 1488 Register SubReg = e == 1 1489 ? ValueReg 1490 : Register(getSubReg(ValueReg, 1491 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1492 1493 unsigned SOffsetRegState = 0; 1494 unsigned SrcDstRegState = getDefRegState(!IsStore); 1495 const bool IsLastSubReg = i + 1 == e; 1496 if (IsLastSubReg) { 1497 SOffsetRegState |= getKillRegState(Scavenged); 1498 // The last implicit use carries the "Kill" flag. 1499 SrcDstRegState |= getKillRegState(IsKill); 1500 } 1501 1502 // Make sure the whole register is defined if there are undef components by 1503 // adding an implicit def of the super-reg on the first instruction. 1504 bool NeedSuperRegDef = e > 1 && IsStore && i == 0; 1505 bool NeedSuperRegImpOperand = e > 1; 1506 1507 // Remaining element size to spill into memory after some parts of it 1508 // spilled into either AGPRs or VGPRs. 1509 unsigned RemEltSize = EltSize; 1510 1511 // AGPRs to spill VGPRs and vice versa are allocated in a reverse order, 1512 // starting from the last lane. In case if a register cannot be completely 1513 // spilled into another register that will ensure its alignment does not 1514 // change. For targets with VGPR alignment requirement this is important 1515 // in case of flat scratch usage as we might get a scratch_load or 1516 // scratch_store of an unaligned register otherwise. 1517 for (int LaneS = (RegOffset + EltSize) / 4 - 1, Lane = LaneS, 1518 LaneE = RegOffset / 4; 1519 Lane >= LaneE; --Lane) { 1520 bool IsSubReg = e > 1 || EltSize > 4; 1521 Register Sub = IsSubReg 1522 ? Register(getSubReg(ValueReg, getSubRegFromChannel(Lane))) 1523 : ValueReg; 1524 auto MIB = spillVGPRtoAGPR(ST, MBB, MI, Index, Lane, Sub, IsKill); 1525 if (!MIB.getInstr()) 1526 break; 1527 if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == LaneS && !i)) { 1528 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1529 NeedSuperRegDef = false; 1530 } 1531 if (IsSubReg || NeedSuperRegImpOperand) { 1532 NeedSuperRegImpOperand = true; 1533 unsigned State = SrcDstRegState; 1534 if (Lane != LaneE) 1535 State &= ~RegState::Kill; 1536 MIB.addReg(ValueReg, RegState::Implicit | State); 1537 } 1538 RemEltSize -= 4; 1539 } 1540 1541 if (!RemEltSize) // Fully spilled into AGPRs. 1542 continue; 1543 1544 if (RemEltSize != EltSize) { // Partially spilled to AGPRs 1545 assert(IsFlat && EltSize > 4); 1546 1547 unsigned NumRegs = RemEltSize / 4; 1548 SubReg = Register(getSubReg(ValueReg, 1549 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1550 unsigned Opc = getFlatScratchSpillOpcode(TII, LoadStoreOp, RemEltSize); 1551 Desc = &TII->get(Opc); 1552 } 1553 1554 unsigned FinalReg = SubReg; 1555 1556 if (IsAGPR) { 1557 assert(EltSize == 4); 1558 1559 if (!TmpIntermediateVGPR) { 1560 assert(MF->getRegInfo().isReserved(AMDGPU::VGPR32)); 1561 TmpIntermediateVGPR = AMDGPU::VGPR32; 1562 } 1563 if (IsStore) { 1564 auto AccRead = BuildMI(MBB, MI, DL, 1565 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), 1566 TmpIntermediateVGPR) 1567 .addReg(SubReg, getKillRegState(IsKill)); 1568 if (NeedSuperRegDef) 1569 AccRead.addReg(ValueReg, RegState::ImplicitDefine); 1570 AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1571 } 1572 SubReg = TmpIntermediateVGPR; 1573 } else if (UseVGPROffset) { 1574 // FIXME: change to scavengeRegisterBackwards() 1575 if (!TmpOffsetVGPR) { 1576 TmpOffsetVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1577 RS->setRegUsed(TmpOffsetVGPR); 1578 } 1579 } 1580 1581 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RegOffset); 1582 MachineMemOperand *NewMMO = 1583 MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize, 1584 commonAlignment(Alignment, RegOffset)); 1585 1586 auto MIB = 1587 BuildMI(MBB, MI, DL, *Desc) 1588 .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill)); 1589 1590 if (UseVGPROffset) { 1591 // For an AGPR spill, we reuse the same temp VGPR for the offset and the 1592 // intermediate accvgpr_write. 1593 MIB.addReg(TmpOffsetVGPR, getKillRegState(IsLastSubReg && !IsAGPR)); 1594 } 1595 1596 if (!IsFlat) 1597 MIB.addReg(FuncInfo->getScratchRSrcReg()); 1598 1599 if (SOffset == AMDGPU::NoRegister) { 1600 if (!IsFlat) { 1601 if (UseVGPROffset && ScratchOffsetReg) { 1602 assert(!FuncInfo->isEntryFunction()); 1603 MIB.addReg(ScratchOffsetReg); 1604 } else { 1605 assert(FuncInfo->isEntryFunction()); 1606 MIB.addImm(0); 1607 } 1608 } 1609 } else { 1610 MIB.addReg(SOffset, SOffsetRegState); 1611 } 1612 MIB.addImm(Offset + RegOffset) 1613 .addImm(0); // cpol 1614 if (!IsFlat) 1615 MIB.addImm(0) // tfe 1616 .addImm(0); // swz 1617 MIB.addMemOperand(NewMMO); 1618 1619 if (!IsAGPR && NeedSuperRegDef) 1620 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1621 1622 if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) { 1623 MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), 1624 FinalReg) 1625 .addReg(TmpIntermediateVGPR, RegState::Kill); 1626 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1627 } 1628 1629 if (NeedSuperRegImpOperand) 1630 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 1631 } 1632 1633 if (ScratchOffsetRegDelta != 0) { 1634 // Subtract the offset we added to the ScratchOffset register. 1635 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1636 .addReg(SOffset) 1637 .addImm(-ScratchOffsetRegDelta); 1638 } 1639 } 1640 1641 void SIRegisterInfo::buildVGPRSpillLoadStore(SGPRSpillBuilder &SB, int Index, 1642 int Offset, bool IsLoad, 1643 bool IsKill) const { 1644 // Load/store VGPR 1645 MachineFrameInfo &FrameInfo = SB.MF.getFrameInfo(); 1646 assert(FrameInfo.getStackID(Index) != TargetStackID::SGPRSpill); 1647 1648 Register FrameReg = 1649 FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(SB.MF) 1650 ? getBaseRegister() 1651 : getFrameRegister(SB.MF); 1652 1653 Align Alignment = FrameInfo.getObjectAlign(Index); 1654 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SB.MF, Index); 1655 MachineMemOperand *MMO = SB.MF.getMachineMemOperand( 1656 PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore, 1657 SB.EltSize, Alignment); 1658 1659 if (IsLoad) { 1660 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 1661 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1662 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, false, 1663 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1664 } else { 1665 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1666 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1667 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, IsKill, 1668 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1669 // This only ever adds one VGPR spill 1670 SB.MFI.addToSpilledVGPRs(1); 1671 } 1672 } 1673 1674 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 1675 int Index, 1676 RegScavenger *RS, 1677 LiveIntervals *LIS, 1678 bool OnlyToVGPR) const { 1679 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1680 1681 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills = 1682 SB.MFI.getSGPRToVGPRSpills(Index); 1683 bool SpillToVGPR = !VGPRSpills.empty(); 1684 if (OnlyToVGPR && !SpillToVGPR) 1685 return false; 1686 1687 assert(SpillToVGPR || (SB.SuperReg != SB.MFI.getStackPtrOffsetReg() && 1688 SB.SuperReg != SB.MFI.getFrameOffsetReg())); 1689 1690 if (SpillToVGPR) { 1691 1692 assert(SB.NumSubRegs == VGPRSpills.size() && 1693 "Num of VGPR lanes should be equal to num of SGPRs spilled"); 1694 1695 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1696 Register SubReg = 1697 SB.NumSubRegs == 1 1698 ? SB.SuperReg 1699 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1700 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 1701 1702 bool UseKill = SB.IsKill && i == SB.NumSubRegs - 1; 1703 1704 // Mark the "old value of vgpr" input undef only if this is the first sgpr 1705 // spill to this specific vgpr in the first basic block. 1706 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1707 SB.TII.get(AMDGPU::V_WRITELANE_B32), Spill.VGPR) 1708 .addReg(SubReg, getKillRegState(UseKill)) 1709 .addImm(Spill.Lane) 1710 .addReg(Spill.VGPR); 1711 if (LIS) { 1712 if (i == 0) 1713 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1714 else 1715 LIS->InsertMachineInstrInMaps(*MIB); 1716 } 1717 1718 if (i == 0 && SB.NumSubRegs > 1) { 1719 // We may be spilling a super-register which is only partially defined, 1720 // and need to ensure later spills think the value is defined. 1721 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1722 } 1723 1724 if (SB.NumSubRegs > 1) 1725 MIB.addReg(SB.SuperReg, getKillRegState(UseKill) | RegState::Implicit); 1726 1727 // FIXME: Since this spills to another register instead of an actual 1728 // frame index, we should delete the frame index when all references to 1729 // it are fixed. 1730 } 1731 } else { 1732 SB.prepare(); 1733 1734 // SubReg carries the "Kill" flag when SubReg == SB.SuperReg. 1735 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1736 1737 // Per VGPR helper data 1738 auto PVD = SB.getPerVGPRData(); 1739 1740 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1741 unsigned TmpVGPRFlags = RegState::Undef; 1742 1743 // Write sub registers into the VGPR 1744 for (unsigned i = Offset * PVD.PerVGPR, 1745 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1746 i < e; ++i) { 1747 Register SubReg = 1748 SB.NumSubRegs == 1 1749 ? SB.SuperReg 1750 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1751 1752 MachineInstrBuilder WriteLane = 1753 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1754 SB.TmpVGPR) 1755 .addReg(SubReg, SubKillState) 1756 .addImm(i % PVD.PerVGPR) 1757 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1758 TmpVGPRFlags = 0; 1759 1760 if (LIS) { 1761 if (i == 0) 1762 LIS->ReplaceMachineInstrInMaps(*MI, *WriteLane); 1763 else 1764 LIS->InsertMachineInstrInMaps(*WriteLane); 1765 } 1766 1767 // There could be undef components of a spilled super register. 1768 // TODO: Can we detect this and skip the spill? 1769 if (SB.NumSubRegs > 1) { 1770 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1771 unsigned SuperKillState = 0; 1772 if (i + 1 == SB.NumSubRegs) 1773 SuperKillState |= getKillRegState(SB.IsKill); 1774 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1775 } 1776 } 1777 1778 // Write out VGPR 1779 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ false); 1780 } 1781 1782 SB.restore(); 1783 } 1784 1785 MI->eraseFromParent(); 1786 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 1787 1788 if (LIS) 1789 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1790 1791 return true; 1792 } 1793 1794 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 1795 int Index, 1796 RegScavenger *RS, 1797 LiveIntervals *LIS, 1798 bool OnlyToVGPR) const { 1799 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1800 1801 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills = 1802 SB.MFI.getSGPRToVGPRSpills(Index); 1803 bool SpillToVGPR = !VGPRSpills.empty(); 1804 if (OnlyToVGPR && !SpillToVGPR) 1805 return false; 1806 1807 if (SpillToVGPR) { 1808 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1809 Register SubReg = 1810 SB.NumSubRegs == 1 1811 ? SB.SuperReg 1812 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1813 1814 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 1815 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1816 SubReg) 1817 .addReg(Spill.VGPR) 1818 .addImm(Spill.Lane); 1819 if (SB.NumSubRegs > 1 && i == 0) 1820 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1821 if (LIS) { 1822 if (i == e - 1) 1823 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1824 else 1825 LIS->InsertMachineInstrInMaps(*MIB); 1826 } 1827 1828 } 1829 } else { 1830 SB.prepare(); 1831 1832 // Per VGPR helper data 1833 auto PVD = SB.getPerVGPRData(); 1834 1835 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1836 // Load in VGPR data 1837 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ true); 1838 1839 // Unpack lanes 1840 for (unsigned i = Offset * PVD.PerVGPR, 1841 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1842 i < e; ++i) { 1843 Register SubReg = 1844 SB.NumSubRegs == 1 1845 ? SB.SuperReg 1846 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1847 1848 bool LastSubReg = (i + 1 == e); 1849 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1850 SB.TII.get(AMDGPU::V_READLANE_B32), SubReg) 1851 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1852 .addImm(i); 1853 if (SB.NumSubRegs > 1 && i == 0) 1854 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1855 if (LIS) { 1856 if (i == e - 1) 1857 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1858 else 1859 LIS->InsertMachineInstrInMaps(*MIB); 1860 } 1861 } 1862 } 1863 1864 SB.restore(); 1865 } 1866 1867 MI->eraseFromParent(); 1868 1869 if (LIS) 1870 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1871 1872 return true; 1873 } 1874 1875 bool SIRegisterInfo::spillEmergencySGPR(MachineBasicBlock::iterator MI, 1876 MachineBasicBlock &RestoreMBB, 1877 Register SGPR, RegScavenger *RS) const { 1878 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, SGPR, false, 0, 1879 RS); 1880 SB.prepare(); 1881 // Generate the spill of SGPR to SB.TmpVGPR. 1882 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1883 auto PVD = SB.getPerVGPRData(); 1884 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1885 unsigned TmpVGPRFlags = RegState::Undef; 1886 // Write sub registers into the VGPR 1887 for (unsigned i = Offset * PVD.PerVGPR, 1888 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1889 i < e; ++i) { 1890 Register SubReg = 1891 SB.NumSubRegs == 1 1892 ? SB.SuperReg 1893 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1894 1895 MachineInstrBuilder WriteLane = 1896 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1897 SB.TmpVGPR) 1898 .addReg(SubReg, SubKillState) 1899 .addImm(i % PVD.PerVGPR) 1900 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1901 TmpVGPRFlags = 0; 1902 // There could be undef components of a spilled super register. 1903 // TODO: Can we detect this and skip the spill? 1904 if (SB.NumSubRegs > 1) { 1905 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1906 unsigned SuperKillState = 0; 1907 if (i + 1 == SB.NumSubRegs) 1908 SuperKillState |= getKillRegState(SB.IsKill); 1909 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1910 } 1911 } 1912 // Don't need to write VGPR out. 1913 } 1914 1915 // Restore clobbered registers in the specified restore block. 1916 MI = RestoreMBB.end(); 1917 SB.setMI(&RestoreMBB, MI); 1918 // Generate the restore of SGPR from SB.TmpVGPR. 1919 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1920 // Don't need to load VGPR in. 1921 // Unpack lanes 1922 for (unsigned i = Offset * PVD.PerVGPR, 1923 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1924 i < e; ++i) { 1925 Register SubReg = 1926 SB.NumSubRegs == 1 1927 ? SB.SuperReg 1928 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1929 bool LastSubReg = (i + 1 == e); 1930 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1931 SubReg) 1932 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1933 .addImm(i); 1934 if (SB.NumSubRegs > 1 && i == 0) 1935 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1936 } 1937 } 1938 SB.restore(); 1939 1940 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 1941 return false; 1942 } 1943 1944 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 1945 /// a VGPR and the stack slot can be safely eliminated when all other users are 1946 /// handled. 1947 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 1948 MachineBasicBlock::iterator MI, 1949 int FI, 1950 RegScavenger *RS, 1951 LiveIntervals *LIS) const { 1952 switch (MI->getOpcode()) { 1953 case AMDGPU::SI_SPILL_S1024_SAVE: 1954 case AMDGPU::SI_SPILL_S512_SAVE: 1955 case AMDGPU::SI_SPILL_S256_SAVE: 1956 case AMDGPU::SI_SPILL_S224_SAVE: 1957 case AMDGPU::SI_SPILL_S192_SAVE: 1958 case AMDGPU::SI_SPILL_S160_SAVE: 1959 case AMDGPU::SI_SPILL_S128_SAVE: 1960 case AMDGPU::SI_SPILL_S96_SAVE: 1961 case AMDGPU::SI_SPILL_S64_SAVE: 1962 case AMDGPU::SI_SPILL_S32_SAVE: 1963 return spillSGPR(MI, FI, RS, LIS, true); 1964 case AMDGPU::SI_SPILL_S1024_RESTORE: 1965 case AMDGPU::SI_SPILL_S512_RESTORE: 1966 case AMDGPU::SI_SPILL_S256_RESTORE: 1967 case AMDGPU::SI_SPILL_S224_RESTORE: 1968 case AMDGPU::SI_SPILL_S192_RESTORE: 1969 case AMDGPU::SI_SPILL_S160_RESTORE: 1970 case AMDGPU::SI_SPILL_S128_RESTORE: 1971 case AMDGPU::SI_SPILL_S96_RESTORE: 1972 case AMDGPU::SI_SPILL_S64_RESTORE: 1973 case AMDGPU::SI_SPILL_S32_RESTORE: 1974 return restoreSGPR(MI, FI, RS, LIS, true); 1975 default: 1976 llvm_unreachable("not an SGPR spill instruction"); 1977 } 1978 } 1979 1980 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1981 int SPAdj, unsigned FIOperandNum, 1982 RegScavenger *RS) const { 1983 MachineFunction *MF = MI->getParent()->getParent(); 1984 MachineBasicBlock *MBB = MI->getParent(); 1985 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1986 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1987 const SIInstrInfo *TII = ST.getInstrInfo(); 1988 DebugLoc DL = MI->getDebugLoc(); 1989 1990 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1991 1992 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1993 int Index = MI->getOperand(FIOperandNum).getIndex(); 1994 1995 Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF) 1996 ? getBaseRegister() 1997 : getFrameRegister(*MF); 1998 1999 switch (MI->getOpcode()) { 2000 // SGPR register spill 2001 case AMDGPU::SI_SPILL_S1024_SAVE: 2002 case AMDGPU::SI_SPILL_S512_SAVE: 2003 case AMDGPU::SI_SPILL_S256_SAVE: 2004 case AMDGPU::SI_SPILL_S224_SAVE: 2005 case AMDGPU::SI_SPILL_S192_SAVE: 2006 case AMDGPU::SI_SPILL_S160_SAVE: 2007 case AMDGPU::SI_SPILL_S128_SAVE: 2008 case AMDGPU::SI_SPILL_S96_SAVE: 2009 case AMDGPU::SI_SPILL_S64_SAVE: 2010 case AMDGPU::SI_SPILL_S32_SAVE: { 2011 spillSGPR(MI, Index, RS); 2012 break; 2013 } 2014 2015 // SGPR register restore 2016 case AMDGPU::SI_SPILL_S1024_RESTORE: 2017 case AMDGPU::SI_SPILL_S512_RESTORE: 2018 case AMDGPU::SI_SPILL_S256_RESTORE: 2019 case AMDGPU::SI_SPILL_S224_RESTORE: 2020 case AMDGPU::SI_SPILL_S192_RESTORE: 2021 case AMDGPU::SI_SPILL_S160_RESTORE: 2022 case AMDGPU::SI_SPILL_S128_RESTORE: 2023 case AMDGPU::SI_SPILL_S96_RESTORE: 2024 case AMDGPU::SI_SPILL_S64_RESTORE: 2025 case AMDGPU::SI_SPILL_S32_RESTORE: { 2026 restoreSGPR(MI, Index, RS); 2027 break; 2028 } 2029 2030 // VGPR register spill 2031 case AMDGPU::SI_SPILL_V1024_SAVE: 2032 case AMDGPU::SI_SPILL_V512_SAVE: 2033 case AMDGPU::SI_SPILL_V256_SAVE: 2034 case AMDGPU::SI_SPILL_V224_SAVE: 2035 case AMDGPU::SI_SPILL_V192_SAVE: 2036 case AMDGPU::SI_SPILL_V160_SAVE: 2037 case AMDGPU::SI_SPILL_V128_SAVE: 2038 case AMDGPU::SI_SPILL_V96_SAVE: 2039 case AMDGPU::SI_SPILL_V64_SAVE: 2040 case AMDGPU::SI_SPILL_V32_SAVE: 2041 case AMDGPU::SI_SPILL_A1024_SAVE: 2042 case AMDGPU::SI_SPILL_A512_SAVE: 2043 case AMDGPU::SI_SPILL_A256_SAVE: 2044 case AMDGPU::SI_SPILL_A224_SAVE: 2045 case AMDGPU::SI_SPILL_A192_SAVE: 2046 case AMDGPU::SI_SPILL_A160_SAVE: 2047 case AMDGPU::SI_SPILL_A128_SAVE: 2048 case AMDGPU::SI_SPILL_A96_SAVE: 2049 case AMDGPU::SI_SPILL_A64_SAVE: 2050 case AMDGPU::SI_SPILL_A32_SAVE: 2051 case AMDGPU::SI_SPILL_AV1024_SAVE: 2052 case AMDGPU::SI_SPILL_AV512_SAVE: 2053 case AMDGPU::SI_SPILL_AV256_SAVE: 2054 case AMDGPU::SI_SPILL_AV224_SAVE: 2055 case AMDGPU::SI_SPILL_AV192_SAVE: 2056 case AMDGPU::SI_SPILL_AV160_SAVE: 2057 case AMDGPU::SI_SPILL_AV128_SAVE: 2058 case AMDGPU::SI_SPILL_AV96_SAVE: 2059 case AMDGPU::SI_SPILL_AV64_SAVE: 2060 case AMDGPU::SI_SPILL_AV32_SAVE: { 2061 const MachineOperand *VData = TII->getNamedOperand(*MI, 2062 AMDGPU::OpName::vdata); 2063 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2064 MFI->getStackPtrOffsetReg()); 2065 2066 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 2067 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 2068 auto *MBB = MI->getParent(); 2069 buildSpillLoadStore( 2070 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2071 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2072 *MI->memoperands_begin(), RS); 2073 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 2074 MI->eraseFromParent(); 2075 break; 2076 } 2077 case AMDGPU::SI_SPILL_V32_RESTORE: 2078 case AMDGPU::SI_SPILL_V64_RESTORE: 2079 case AMDGPU::SI_SPILL_V96_RESTORE: 2080 case AMDGPU::SI_SPILL_V128_RESTORE: 2081 case AMDGPU::SI_SPILL_V160_RESTORE: 2082 case AMDGPU::SI_SPILL_V192_RESTORE: 2083 case AMDGPU::SI_SPILL_V224_RESTORE: 2084 case AMDGPU::SI_SPILL_V256_RESTORE: 2085 case AMDGPU::SI_SPILL_V512_RESTORE: 2086 case AMDGPU::SI_SPILL_V1024_RESTORE: 2087 case AMDGPU::SI_SPILL_A32_RESTORE: 2088 case AMDGPU::SI_SPILL_A64_RESTORE: 2089 case AMDGPU::SI_SPILL_A96_RESTORE: 2090 case AMDGPU::SI_SPILL_A128_RESTORE: 2091 case AMDGPU::SI_SPILL_A160_RESTORE: 2092 case AMDGPU::SI_SPILL_A192_RESTORE: 2093 case AMDGPU::SI_SPILL_A224_RESTORE: 2094 case AMDGPU::SI_SPILL_A256_RESTORE: 2095 case AMDGPU::SI_SPILL_A512_RESTORE: 2096 case AMDGPU::SI_SPILL_A1024_RESTORE: 2097 case AMDGPU::SI_SPILL_AV32_RESTORE: 2098 case AMDGPU::SI_SPILL_AV64_RESTORE: 2099 case AMDGPU::SI_SPILL_AV96_RESTORE: 2100 case AMDGPU::SI_SPILL_AV128_RESTORE: 2101 case AMDGPU::SI_SPILL_AV160_RESTORE: 2102 case AMDGPU::SI_SPILL_AV192_RESTORE: 2103 case AMDGPU::SI_SPILL_AV224_RESTORE: 2104 case AMDGPU::SI_SPILL_AV256_RESTORE: 2105 case AMDGPU::SI_SPILL_AV512_RESTORE: 2106 case AMDGPU::SI_SPILL_AV1024_RESTORE: { 2107 const MachineOperand *VData = TII->getNamedOperand(*MI, 2108 AMDGPU::OpName::vdata); 2109 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2110 MFI->getStackPtrOffsetReg()); 2111 2112 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 2113 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 2114 auto *MBB = MI->getParent(); 2115 buildSpillLoadStore( 2116 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2117 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2118 *MI->memoperands_begin(), RS); 2119 MI->eraseFromParent(); 2120 break; 2121 } 2122 2123 default: { 2124 // Other access to frame index 2125 const DebugLoc &DL = MI->getDebugLoc(); 2126 2127 int64_t Offset = FrameInfo.getObjectOffset(Index); 2128 if (ST.enableFlatScratch()) { 2129 if (TII->isFLATScratch(*MI)) { 2130 assert((int16_t)FIOperandNum == 2131 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2132 AMDGPU::OpName::saddr)); 2133 2134 // The offset is always swizzled, just replace it 2135 if (FrameReg) 2136 FIOp.ChangeToRegister(FrameReg, false); 2137 2138 if (!Offset) 2139 return; 2140 2141 MachineOperand *OffsetOp = 2142 TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 2143 int64_t NewOffset = Offset + OffsetOp->getImm(); 2144 if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 2145 SIInstrFlags::FlatScratch)) { 2146 OffsetOp->setImm(NewOffset); 2147 if (FrameReg) 2148 return; 2149 Offset = 0; 2150 } 2151 2152 assert(!TII->getNamedOperand(*MI, AMDGPU::OpName::vaddr) && 2153 "Unexpected vaddr for flat scratch with a FI operand"); 2154 2155 // On GFX10 we have ST mode to use no registers for an address. 2156 // Otherwise we need to materialize 0 into an SGPR. 2157 if (!Offset && ST.hasFlatScratchSTMode()) { 2158 unsigned Opc = MI->getOpcode(); 2159 unsigned NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc); 2160 MI->RemoveOperand( 2161 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr)); 2162 MI->setDesc(TII->get(NewOpc)); 2163 return; 2164 } 2165 } 2166 2167 if (!FrameReg) { 2168 FIOp.ChangeToImmediate(Offset); 2169 if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) 2170 return; 2171 } 2172 2173 // We need to use register here. Check if we can use an SGPR or need 2174 // a VGPR. 2175 FIOp.ChangeToRegister(AMDGPU::M0, false); 2176 bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, &FIOp); 2177 2178 if (!Offset && FrameReg && UseSGPR) { 2179 FIOp.setReg(FrameReg); 2180 return; 2181 } 2182 2183 const TargetRegisterClass *RC = UseSGPR ? &AMDGPU::SReg_32_XM0RegClass 2184 : &AMDGPU::VGPR_32RegClass; 2185 2186 Register TmpReg = RS->scavengeRegister(RC, MI, 0, !UseSGPR); 2187 FIOp.setReg(TmpReg); 2188 FIOp.setIsKill(true); 2189 2190 if ((!FrameReg || !Offset) && TmpReg) { 2191 unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 2192 auto MIB = BuildMI(*MBB, MI, DL, TII->get(Opc), TmpReg); 2193 if (FrameReg) 2194 MIB.addReg(FrameReg); 2195 else 2196 MIB.addImm(Offset); 2197 2198 return; 2199 } 2200 2201 Register TmpSReg = 2202 UseSGPR ? TmpReg 2203 : RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, 2204 !UseSGPR); 2205 2206 // TODO: for flat scratch another attempt can be made with a VGPR index 2207 // if no SGPRs can be scavenged. 2208 if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR)) 2209 report_fatal_error("Cannot scavenge register in FI elimination!"); 2210 2211 if (!TmpSReg) { 2212 // Use frame register and restore it after. 2213 TmpSReg = FrameReg; 2214 FIOp.setReg(FrameReg); 2215 FIOp.setIsKill(false); 2216 } 2217 2218 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpSReg) 2219 .addReg(FrameReg) 2220 .addImm(Offset); 2221 2222 if (!UseSGPR) 2223 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2224 .addReg(TmpSReg, RegState::Kill); 2225 2226 if (TmpSReg == FrameReg) { 2227 // Undo frame register modification. 2228 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADD_I32), 2229 FrameReg) 2230 .addReg(FrameReg) 2231 .addImm(-Offset); 2232 } 2233 2234 return; 2235 } 2236 2237 bool IsMUBUF = TII->isMUBUF(*MI); 2238 2239 if (!IsMUBUF && !MFI->isEntryFunction()) { 2240 // Convert to a swizzled stack address by scaling by the wave size. 2241 // 2242 // In an entry function/kernel the offset is already swizzled. 2243 2244 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 2245 Register ResultReg = 2246 IsCopy ? MI->getOperand(0).getReg() 2247 : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 2248 2249 int64_t Offset = FrameInfo.getObjectOffset(Index); 2250 if (Offset == 0) { 2251 // XXX - This never happens because of emergency scavenging slot at 0? 2252 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 2253 .addImm(ST.getWavefrontSizeLog2()) 2254 .addReg(FrameReg); 2255 } else { 2256 if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) { 2257 // Reuse ResultReg in intermediate step. 2258 Register ScaledReg = ResultReg; 2259 2260 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 2261 ScaledReg) 2262 .addImm(ST.getWavefrontSizeLog2()) 2263 .addReg(FrameReg); 2264 2265 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 2266 2267 // TODO: Fold if use instruction is another add of a constant. 2268 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 2269 // FIXME: This can fail 2270 MIB.addImm(Offset); 2271 MIB.addReg(ScaledReg, RegState::Kill); 2272 if (!IsVOP2) 2273 MIB.addImm(0); // clamp bit 2274 } else { 2275 assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 && 2276 "Need to reuse carry out register"); 2277 2278 // Use scavenged unused carry out as offset register. 2279 Register ConstOffsetReg; 2280 if (!isWave32) 2281 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 2282 else 2283 ConstOffsetReg = MIB.getReg(1); 2284 2285 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 2286 .addImm(Offset); 2287 MIB.addReg(ConstOffsetReg, RegState::Kill); 2288 MIB.addReg(ScaledReg, RegState::Kill); 2289 MIB.addImm(0); // clamp bit 2290 } 2291 } else { 2292 // We have to produce a carry out, and there isn't a free SGPR pair 2293 // for it. We can keep the whole computation on the SALU to avoid 2294 // clobbering an additional register at the cost of an extra mov. 2295 2296 // We may have 1 free scratch SGPR even though a carry out is 2297 // unavailable. Only one additional mov is needed. 2298 Register TmpScaledReg = 2299 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 2300 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 2301 2302 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 2303 .addReg(FrameReg) 2304 .addImm(ST.getWavefrontSizeLog2()); 2305 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2306 .addReg(ScaledReg, RegState::Kill) 2307 .addImm(Offset); 2308 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 2309 .addReg(ScaledReg, RegState::Kill); 2310 2311 // If there were truly no free SGPRs, we need to undo everything. 2312 if (!TmpScaledReg.isValid()) { 2313 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2314 .addReg(ScaledReg, RegState::Kill) 2315 .addImm(-Offset); 2316 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 2317 .addReg(FrameReg) 2318 .addImm(ST.getWavefrontSizeLog2()); 2319 } 2320 } 2321 } 2322 2323 // Don't introduce an extra copy if we're just materializing in a mov. 2324 if (IsCopy) 2325 MI->eraseFromParent(); 2326 else 2327 FIOp.ChangeToRegister(ResultReg, false, false, true); 2328 return; 2329 } 2330 2331 if (IsMUBUF) { 2332 // Disable offen so we don't need a 0 vgpr base. 2333 assert(static_cast<int>(FIOperandNum) == 2334 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2335 AMDGPU::OpName::vaddr)); 2336 2337 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 2338 assert((SOffset.isImm() && SOffset.getImm() == 0)); 2339 2340 if (FrameReg != AMDGPU::NoRegister) 2341 SOffset.ChangeToRegister(FrameReg, false); 2342 2343 int64_t Offset = FrameInfo.getObjectOffset(Index); 2344 int64_t OldImm 2345 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 2346 int64_t NewOffset = OldImm + Offset; 2347 2348 if (SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 2349 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 2350 MI->eraseFromParent(); 2351 return; 2352 } 2353 } 2354 2355 // If the offset is simply too big, don't convert to a scratch wave offset 2356 // relative index. 2357 2358 FIOp.ChangeToImmediate(Offset); 2359 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 2360 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 2361 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2362 .addImm(Offset); 2363 FIOp.ChangeToRegister(TmpReg, false, false, true); 2364 } 2365 } 2366 } 2367 } 2368 2369 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 2370 return AMDGPUInstPrinter::getRegisterName(Reg); 2371 } 2372 2373 static const TargetRegisterClass * 2374 getAnyVGPRClassForBitWidth(unsigned BitWidth) { 2375 if (BitWidth <= 64) 2376 return &AMDGPU::VReg_64RegClass; 2377 if (BitWidth <= 96) 2378 return &AMDGPU::VReg_96RegClass; 2379 if (BitWidth <= 128) 2380 return &AMDGPU::VReg_128RegClass; 2381 if (BitWidth <= 160) 2382 return &AMDGPU::VReg_160RegClass; 2383 if (BitWidth <= 192) 2384 return &AMDGPU::VReg_192RegClass; 2385 if (BitWidth <= 224) 2386 return &AMDGPU::VReg_224RegClass; 2387 if (BitWidth <= 256) 2388 return &AMDGPU::VReg_256RegClass; 2389 if (BitWidth <= 512) 2390 return &AMDGPU::VReg_512RegClass; 2391 if (BitWidth <= 1024) 2392 return &AMDGPU::VReg_1024RegClass; 2393 2394 return nullptr; 2395 } 2396 2397 static const TargetRegisterClass * 2398 getAlignedVGPRClassForBitWidth(unsigned BitWidth) { 2399 if (BitWidth <= 64) 2400 return &AMDGPU::VReg_64_Align2RegClass; 2401 if (BitWidth <= 96) 2402 return &AMDGPU::VReg_96_Align2RegClass; 2403 if (BitWidth <= 128) 2404 return &AMDGPU::VReg_128_Align2RegClass; 2405 if (BitWidth <= 160) 2406 return &AMDGPU::VReg_160_Align2RegClass; 2407 if (BitWidth <= 192) 2408 return &AMDGPU::VReg_192_Align2RegClass; 2409 if (BitWidth <= 224) 2410 return &AMDGPU::VReg_224_Align2RegClass; 2411 if (BitWidth <= 256) 2412 return &AMDGPU::VReg_256_Align2RegClass; 2413 if (BitWidth <= 512) 2414 return &AMDGPU::VReg_512_Align2RegClass; 2415 if (BitWidth <= 1024) 2416 return &AMDGPU::VReg_1024_Align2RegClass; 2417 2418 return nullptr; 2419 } 2420 2421 const TargetRegisterClass * 2422 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const { 2423 if (BitWidth == 1) 2424 return &AMDGPU::VReg_1RegClass; 2425 if (BitWidth <= 16) 2426 return &AMDGPU::VGPR_LO16RegClass; 2427 if (BitWidth <= 32) 2428 return &AMDGPU::VGPR_32RegClass; 2429 return ST.needsAlignedVGPRs() ? getAlignedVGPRClassForBitWidth(BitWidth) 2430 : getAnyVGPRClassForBitWidth(BitWidth); 2431 } 2432 2433 static const TargetRegisterClass * 2434 getAnyAGPRClassForBitWidth(unsigned BitWidth) { 2435 if (BitWidth <= 64) 2436 return &AMDGPU::AReg_64RegClass; 2437 if (BitWidth <= 96) 2438 return &AMDGPU::AReg_96RegClass; 2439 if (BitWidth <= 128) 2440 return &AMDGPU::AReg_128RegClass; 2441 if (BitWidth <= 160) 2442 return &AMDGPU::AReg_160RegClass; 2443 if (BitWidth <= 192) 2444 return &AMDGPU::AReg_192RegClass; 2445 if (BitWidth <= 224) 2446 return &AMDGPU::AReg_224RegClass; 2447 if (BitWidth <= 256) 2448 return &AMDGPU::AReg_256RegClass; 2449 if (BitWidth <= 512) 2450 return &AMDGPU::AReg_512RegClass; 2451 if (BitWidth <= 1024) 2452 return &AMDGPU::AReg_1024RegClass; 2453 2454 return nullptr; 2455 } 2456 2457 static const TargetRegisterClass * 2458 getAlignedAGPRClassForBitWidth(unsigned BitWidth) { 2459 if (BitWidth <= 64) 2460 return &AMDGPU::AReg_64_Align2RegClass; 2461 if (BitWidth <= 96) 2462 return &AMDGPU::AReg_96_Align2RegClass; 2463 if (BitWidth <= 128) 2464 return &AMDGPU::AReg_128_Align2RegClass; 2465 if (BitWidth <= 160) 2466 return &AMDGPU::AReg_160_Align2RegClass; 2467 if (BitWidth <= 192) 2468 return &AMDGPU::AReg_192_Align2RegClass; 2469 if (BitWidth <= 224) 2470 return &AMDGPU::AReg_224_Align2RegClass; 2471 if (BitWidth <= 256) 2472 return &AMDGPU::AReg_256_Align2RegClass; 2473 if (BitWidth <= 512) 2474 return &AMDGPU::AReg_512_Align2RegClass; 2475 if (BitWidth <= 1024) 2476 return &AMDGPU::AReg_1024_Align2RegClass; 2477 2478 return nullptr; 2479 } 2480 2481 const TargetRegisterClass * 2482 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const { 2483 if (BitWidth <= 16) 2484 return &AMDGPU::AGPR_LO16RegClass; 2485 if (BitWidth <= 32) 2486 return &AMDGPU::AGPR_32RegClass; 2487 return ST.needsAlignedVGPRs() ? getAlignedAGPRClassForBitWidth(BitWidth) 2488 : getAnyAGPRClassForBitWidth(BitWidth); 2489 } 2490 2491 static const TargetRegisterClass * 2492 getAnyVectorSuperClassForBitWidth(unsigned BitWidth) { 2493 if (BitWidth <= 64) 2494 return &AMDGPU::AV_64RegClass; 2495 if (BitWidth <= 96) 2496 return &AMDGPU::AV_96RegClass; 2497 if (BitWidth <= 128) 2498 return &AMDGPU::AV_128RegClass; 2499 if (BitWidth <= 160) 2500 return &AMDGPU::AV_160RegClass; 2501 if (BitWidth <= 192) 2502 return &AMDGPU::AV_192RegClass; 2503 if (BitWidth <= 224) 2504 return &AMDGPU::AV_224RegClass; 2505 if (BitWidth <= 256) 2506 return &AMDGPU::AV_256RegClass; 2507 if (BitWidth <= 512) 2508 return &AMDGPU::AV_512RegClass; 2509 if (BitWidth <= 1024) 2510 return &AMDGPU::AV_1024RegClass; 2511 2512 return nullptr; 2513 } 2514 2515 static const TargetRegisterClass * 2516 getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) { 2517 if (BitWidth <= 64) 2518 return &AMDGPU::AV_64_Align2RegClass; 2519 if (BitWidth <= 96) 2520 return &AMDGPU::AV_96_Align2RegClass; 2521 if (BitWidth <= 128) 2522 return &AMDGPU::AV_128_Align2RegClass; 2523 if (BitWidth <= 160) 2524 return &AMDGPU::AV_160_Align2RegClass; 2525 if (BitWidth <= 192) 2526 return &AMDGPU::AV_192_Align2RegClass; 2527 if (BitWidth <= 224) 2528 return &AMDGPU::AV_224_Align2RegClass; 2529 if (BitWidth <= 256) 2530 return &AMDGPU::AV_256_Align2RegClass; 2531 if (BitWidth <= 512) 2532 return &AMDGPU::AV_512_Align2RegClass; 2533 if (BitWidth <= 1024) 2534 return &AMDGPU::AV_1024_Align2RegClass; 2535 2536 return nullptr; 2537 } 2538 2539 const TargetRegisterClass * 2540 SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const { 2541 if (BitWidth <= 16) 2542 return &AMDGPU::VGPR_LO16RegClass; 2543 if (BitWidth <= 32) 2544 return &AMDGPU::AV_32RegClass; 2545 return ST.needsAlignedVGPRs() 2546 ? getAlignedVectorSuperClassForBitWidth(BitWidth) 2547 : getAnyVectorSuperClassForBitWidth(BitWidth); 2548 } 2549 2550 const TargetRegisterClass * 2551 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 2552 if (BitWidth <= 16) 2553 return &AMDGPU::SGPR_LO16RegClass; 2554 if (BitWidth <= 32) 2555 return &AMDGPU::SReg_32RegClass; 2556 if (BitWidth <= 64) 2557 return &AMDGPU::SReg_64RegClass; 2558 if (BitWidth <= 96) 2559 return &AMDGPU::SGPR_96RegClass; 2560 if (BitWidth <= 128) 2561 return &AMDGPU::SGPR_128RegClass; 2562 if (BitWidth <= 160) 2563 return &AMDGPU::SGPR_160RegClass; 2564 if (BitWidth <= 192) 2565 return &AMDGPU::SGPR_192RegClass; 2566 if (BitWidth <= 224) 2567 return &AMDGPU::SGPR_224RegClass; 2568 if (BitWidth <= 256) 2569 return &AMDGPU::SGPR_256RegClass; 2570 if (BitWidth <= 512) 2571 return &AMDGPU::SGPR_512RegClass; 2572 if (BitWidth <= 1024) 2573 return &AMDGPU::SGPR_1024RegClass; 2574 2575 return nullptr; 2576 } 2577 2578 // FIXME: This is very slow. It might be worth creating a map from physreg to 2579 // register class. 2580 const TargetRegisterClass * 2581 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const { 2582 static const TargetRegisterClass *const BaseClasses[] = { 2583 &AMDGPU::VGPR_LO16RegClass, 2584 &AMDGPU::VGPR_HI16RegClass, 2585 &AMDGPU::SReg_LO16RegClass, 2586 &AMDGPU::AGPR_LO16RegClass, 2587 &AMDGPU::VGPR_32RegClass, 2588 &AMDGPU::SReg_32RegClass, 2589 &AMDGPU::AGPR_32RegClass, 2590 &AMDGPU::AGPR_32RegClass, 2591 &AMDGPU::VReg_64_Align2RegClass, 2592 &AMDGPU::VReg_64RegClass, 2593 &AMDGPU::SReg_64RegClass, 2594 &AMDGPU::AReg_64_Align2RegClass, 2595 &AMDGPU::AReg_64RegClass, 2596 &AMDGPU::VReg_96_Align2RegClass, 2597 &AMDGPU::VReg_96RegClass, 2598 &AMDGPU::SReg_96RegClass, 2599 &AMDGPU::AReg_96_Align2RegClass, 2600 &AMDGPU::AReg_96RegClass, 2601 &AMDGPU::VReg_128_Align2RegClass, 2602 &AMDGPU::VReg_128RegClass, 2603 &AMDGPU::SReg_128RegClass, 2604 &AMDGPU::AReg_128_Align2RegClass, 2605 &AMDGPU::AReg_128RegClass, 2606 &AMDGPU::VReg_160_Align2RegClass, 2607 &AMDGPU::VReg_160RegClass, 2608 &AMDGPU::SReg_160RegClass, 2609 &AMDGPU::AReg_160_Align2RegClass, 2610 &AMDGPU::AReg_160RegClass, 2611 &AMDGPU::VReg_192_Align2RegClass, 2612 &AMDGPU::VReg_192RegClass, 2613 &AMDGPU::SReg_192RegClass, 2614 &AMDGPU::AReg_192_Align2RegClass, 2615 &AMDGPU::AReg_192RegClass, 2616 &AMDGPU::VReg_224_Align2RegClass, 2617 &AMDGPU::VReg_224RegClass, 2618 &AMDGPU::SReg_224RegClass, 2619 &AMDGPU::AReg_224_Align2RegClass, 2620 &AMDGPU::AReg_224RegClass, 2621 &AMDGPU::VReg_256_Align2RegClass, 2622 &AMDGPU::VReg_256RegClass, 2623 &AMDGPU::SReg_256RegClass, 2624 &AMDGPU::AReg_256_Align2RegClass, 2625 &AMDGPU::AReg_256RegClass, 2626 &AMDGPU::VReg_512_Align2RegClass, 2627 &AMDGPU::VReg_512RegClass, 2628 &AMDGPU::SReg_512RegClass, 2629 &AMDGPU::AReg_512_Align2RegClass, 2630 &AMDGPU::AReg_512RegClass, 2631 &AMDGPU::SReg_1024RegClass, 2632 &AMDGPU::VReg_1024_Align2RegClass, 2633 &AMDGPU::VReg_1024RegClass, 2634 &AMDGPU::AReg_1024_Align2RegClass, 2635 &AMDGPU::AReg_1024RegClass, 2636 &AMDGPU::SCC_CLASSRegClass, 2637 &AMDGPU::Pseudo_SReg_32RegClass, 2638 &AMDGPU::Pseudo_SReg_128RegClass, 2639 }; 2640 2641 for (const TargetRegisterClass *BaseClass : BaseClasses) { 2642 if (BaseClass->contains(Reg)) { 2643 return BaseClass; 2644 } 2645 } 2646 return nullptr; 2647 } 2648 2649 bool SIRegisterInfo::isSGPRReg(const MachineRegisterInfo &MRI, 2650 Register Reg) const { 2651 const TargetRegisterClass *RC; 2652 if (Reg.isVirtual()) 2653 RC = MRI.getRegClass(Reg); 2654 else 2655 RC = getPhysRegClass(Reg); 2656 return isSGPRClass(RC); 2657 } 2658 2659 const TargetRegisterClass * 2660 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 2661 unsigned Size = getRegSizeInBits(*SRC); 2662 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 2663 assert(VRC && "Invalid register class size"); 2664 return VRC; 2665 } 2666 2667 const TargetRegisterClass * 2668 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 2669 unsigned Size = getRegSizeInBits(*SRC); 2670 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 2671 assert(ARC && "Invalid register class size"); 2672 return ARC; 2673 } 2674 2675 const TargetRegisterClass * 2676 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 2677 unsigned Size = getRegSizeInBits(*VRC); 2678 if (Size == 32) 2679 return &AMDGPU::SGPR_32RegClass; 2680 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 2681 assert(SRC && "Invalid register class size"); 2682 return SRC; 2683 } 2684 2685 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 2686 const TargetRegisterClass *RC, unsigned SubIdx) const { 2687 if (SubIdx == AMDGPU::NoSubRegister) 2688 return RC; 2689 2690 // We can assume that each lane corresponds to one 32-bit register. 2691 unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32; 2692 if (isAGPRClass(RC)) { 2693 RC = getAGPRClassForBitWidth(Size); 2694 } else if (isVGPRClass(RC)) { 2695 RC = getVGPRClassForBitWidth(Size); 2696 } else if (isVectorSuperClass(RC)) { 2697 RC = getVectorSuperClassForBitWidth(Size); 2698 } else { 2699 RC = getSGPRClassForBitWidth(Size); 2700 } 2701 assert(RC && "Invalid sub-register class size"); 2702 return RC; 2703 } 2704 2705 const TargetRegisterClass * 2706 SIRegisterInfo::getCompatibleSubRegClass(const TargetRegisterClass *SuperRC, 2707 const TargetRegisterClass *SubRC, 2708 unsigned SubIdx) const { 2709 // Ensure this subregister index is aligned in the super register. 2710 const TargetRegisterClass *MatchRC = 2711 getMatchingSuperRegClass(SuperRC, SubRC, SubIdx); 2712 return MatchRC && MatchRC->hasSubClassEq(SuperRC) ? MatchRC : nullptr; 2713 } 2714 2715 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 2716 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 2717 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 2718 return !ST.hasMFMAInlineLiteralBug(); 2719 2720 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 2721 OpType <= AMDGPU::OPERAND_SRC_LAST; 2722 } 2723 2724 bool SIRegisterInfo::shouldRewriteCopySrc( 2725 const TargetRegisterClass *DefRC, 2726 unsigned DefSubReg, 2727 const TargetRegisterClass *SrcRC, 2728 unsigned SrcSubReg) const { 2729 // We want to prefer the smallest register class possible, so we don't want to 2730 // stop and rewrite on anything that looks like a subregister 2731 // extract. Operations mostly don't care about the super register class, so we 2732 // only want to stop on the most basic of copies between the same register 2733 // class. 2734 // 2735 // e.g. if we have something like 2736 // %0 = ... 2737 // %1 = ... 2738 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 2739 // %3 = COPY %2, sub0 2740 // 2741 // We want to look through the COPY to find: 2742 // => %3 = COPY %0 2743 2744 // Plain copy. 2745 return getCommonSubClass(DefRC, SrcRC) != nullptr; 2746 } 2747 2748 bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const { 2749 // TODO: 64-bit operands have extending behavior from 32-bit literal. 2750 return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST && 2751 OpType <= AMDGPU::OPERAND_REG_IMM_LAST; 2752 } 2753 2754 /// Returns a lowest register that is not used at any point in the function. 2755 /// If all registers are used, then this function will return 2756 /// AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return 2757 /// highest unused register. 2758 MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 2759 const TargetRegisterClass *RC, 2760 const MachineFunction &MF, 2761 bool ReserveHighestVGPR) const { 2762 if (ReserveHighestVGPR) { 2763 for (MCRegister Reg : reverse(*RC)) 2764 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2765 return Reg; 2766 } else { 2767 for (MCRegister Reg : *RC) 2768 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2769 return Reg; 2770 } 2771 return MCRegister(); 2772 } 2773 2774 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 2775 unsigned EltSize) const { 2776 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC); 2777 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 2778 2779 const unsigned RegDWORDs = RegBitWidth / 32; 2780 const unsigned EltDWORDs = EltSize / 4; 2781 assert(RegSplitParts.size() + 1 >= EltDWORDs); 2782 2783 const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1]; 2784 const unsigned NumParts = RegDWORDs / EltDWORDs; 2785 2786 return makeArrayRef(Parts.data(), NumParts); 2787 } 2788 2789 const TargetRegisterClass* 2790 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 2791 Register Reg) const { 2792 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg); 2793 } 2794 2795 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 2796 Register Reg) const { 2797 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2798 // Registers without classes are unaddressable, SGPR-like registers. 2799 return RC && isVGPRClass(RC); 2800 } 2801 2802 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 2803 Register Reg) const { 2804 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2805 2806 // Registers without classes are unaddressable, SGPR-like registers. 2807 return RC && isAGPRClass(RC); 2808 } 2809 2810 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 2811 const TargetRegisterClass *SrcRC, 2812 unsigned SubReg, 2813 const TargetRegisterClass *DstRC, 2814 unsigned DstSubReg, 2815 const TargetRegisterClass *NewRC, 2816 LiveIntervals &LIS) const { 2817 unsigned SrcSize = getRegSizeInBits(*SrcRC); 2818 unsigned DstSize = getRegSizeInBits(*DstRC); 2819 unsigned NewSize = getRegSizeInBits(*NewRC); 2820 2821 // Do not increase size of registers beyond dword, we would need to allocate 2822 // adjacent registers and constraint regalloc more than needed. 2823 2824 // Always allow dword coalescing. 2825 if (SrcSize <= 32 || DstSize <= 32) 2826 return true; 2827 2828 return NewSize <= DstSize || NewSize <= SrcSize; 2829 } 2830 2831 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 2832 MachineFunction &MF) const { 2833 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 2834 2835 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 2836 MF.getFunction()); 2837 switch (RC->getID()) { 2838 default: 2839 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 2840 case AMDGPU::VGPR_32RegClassID: 2841 case AMDGPU::VGPR_LO16RegClassID: 2842 case AMDGPU::VGPR_HI16RegClassID: 2843 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 2844 case AMDGPU::SGPR_32RegClassID: 2845 case AMDGPU::SGPR_LO16RegClassID: 2846 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 2847 } 2848 } 2849 2850 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 2851 unsigned Idx) const { 2852 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 2853 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 2854 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 2855 const_cast<MachineFunction &>(MF)); 2856 2857 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 2858 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 2859 const_cast<MachineFunction &>(MF)); 2860 2861 llvm_unreachable("Unexpected register pressure set!"); 2862 } 2863 2864 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 2865 static const int Empty[] = { -1 }; 2866 2867 if (RegPressureIgnoredUnits[RegUnit]) 2868 return Empty; 2869 2870 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 2871 } 2872 2873 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 2874 // Not a callee saved register. 2875 return AMDGPU::SGPR30_SGPR31; 2876 } 2877 2878 const TargetRegisterClass * 2879 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 2880 const RegisterBank &RB, 2881 const MachineRegisterInfo &MRI) const { 2882 switch (RB.getID()) { 2883 case AMDGPU::VGPRRegBankID: 2884 return getVGPRClassForBitWidth(std::max(32u, Size)); 2885 case AMDGPU::VCCRegBankID: 2886 assert(Size == 1); 2887 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 2888 : &AMDGPU::SReg_64_XEXECRegClass; 2889 case AMDGPU::SGPRRegBankID: 2890 return getSGPRClassForBitWidth(std::max(32u, Size)); 2891 case AMDGPU::AGPRRegBankID: 2892 return getAGPRClassForBitWidth(std::max(32u, Size)); 2893 default: 2894 llvm_unreachable("unknown register bank"); 2895 } 2896 } 2897 2898 const TargetRegisterClass * 2899 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 2900 const MachineRegisterInfo &MRI) const { 2901 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 2902 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 2903 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI); 2904 2905 if (const auto *RC = RCOrRB.dyn_cast<const TargetRegisterClass *>()) 2906 return getAllocatableClass(RC); 2907 2908 return nullptr; 2909 } 2910 2911 MCRegister SIRegisterInfo::getVCC() const { 2912 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 2913 } 2914 2915 const TargetRegisterClass *SIRegisterInfo::getVGPR64Class() const { 2916 // VGPR tuples have an alignment requirement on gfx90a variants. 2917 return ST.needsAlignedVGPRs() ? &AMDGPU::VReg_64_Align2RegClass 2918 : &AMDGPU::VReg_64RegClass; 2919 } 2920 2921 const TargetRegisterClass * 2922 SIRegisterInfo::getRegClass(unsigned RCID) const { 2923 switch ((int)RCID) { 2924 case AMDGPU::SReg_1RegClassID: 2925 return getBoolRC(); 2926 case AMDGPU::SReg_1_XEXECRegClassID: 2927 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 2928 : &AMDGPU::SReg_64_XEXECRegClass; 2929 case -1: 2930 return nullptr; 2931 default: 2932 return AMDGPUGenRegisterInfo::getRegClass(RCID); 2933 } 2934 } 2935 2936 // Find reaching register definition 2937 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 2938 MachineInstr &Use, 2939 MachineRegisterInfo &MRI, 2940 LiveIntervals *LIS) const { 2941 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 2942 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 2943 SlotIndex DefIdx; 2944 2945 if (Reg.isVirtual()) { 2946 if (!LIS->hasInterval(Reg)) 2947 return nullptr; 2948 LiveInterval &LI = LIS->getInterval(Reg); 2949 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 2950 : MRI.getMaxLaneMaskForVReg(Reg); 2951 VNInfo *V = nullptr; 2952 if (LI.hasSubRanges()) { 2953 for (auto &S : LI.subranges()) { 2954 if ((S.LaneMask & SubLanes) == SubLanes) { 2955 V = S.getVNInfoAt(UseIdx); 2956 break; 2957 } 2958 } 2959 } else { 2960 V = LI.getVNInfoAt(UseIdx); 2961 } 2962 if (!V) 2963 return nullptr; 2964 DefIdx = V->def; 2965 } else { 2966 // Find last def. 2967 for (MCRegUnitIterator Units(Reg.asMCReg(), this); Units.isValid(); 2968 ++Units) { 2969 LiveRange &LR = LIS->getRegUnit(*Units); 2970 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 2971 if (!DefIdx.isValid() || 2972 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 2973 LIS->getInstructionFromIndex(V->def))) 2974 DefIdx = V->def; 2975 } else { 2976 return nullptr; 2977 } 2978 } 2979 } 2980 2981 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 2982 2983 if (!Def || !MDT.dominates(Def, &Use)) 2984 return nullptr; 2985 2986 assert(Def->modifiesRegister(Reg, this)); 2987 2988 return Def; 2989 } 2990 2991 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 2992 assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32); 2993 2994 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 2995 AMDGPU::SReg_32RegClass, 2996 AMDGPU::AGPR_32RegClass } ) { 2997 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 2998 return Super; 2999 } 3000 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 3001 &AMDGPU::VGPR_32RegClass)) { 3002 return Super; 3003 } 3004 3005 return AMDGPU::NoRegister; 3006 } 3007 3008 bool SIRegisterInfo::isProperlyAlignedRC(const TargetRegisterClass &RC) const { 3009 if (!ST.needsAlignedVGPRs()) 3010 return true; 3011 3012 if (isVGPRClass(&RC)) 3013 return RC.hasSuperClassEq(getVGPRClassForBitWidth(getRegSizeInBits(RC))); 3014 if (isAGPRClass(&RC)) 3015 return RC.hasSuperClassEq(getAGPRClassForBitWidth(getRegSizeInBits(RC))); 3016 if (isVectorSuperClass(&RC)) 3017 return RC.hasSuperClassEq( 3018 getVectorSuperClassForBitWidth(getRegSizeInBits(RC))); 3019 3020 return true; 3021 } 3022 3023 bool SIRegisterInfo::isConstantPhysReg(MCRegister PhysReg) const { 3024 switch (PhysReg) { 3025 case AMDGPU::SGPR_NULL: 3026 case AMDGPU::SRC_SHARED_BASE: 3027 case AMDGPU::SRC_PRIVATE_BASE: 3028 case AMDGPU::SRC_SHARED_LIMIT: 3029 case AMDGPU::SRC_PRIVATE_LIMIT: 3030 return true; 3031 default: 3032 return false; 3033 } 3034 } 3035 3036 ArrayRef<MCPhysReg> 3037 SIRegisterInfo::getAllSGPR128(const MachineFunction &MF) const { 3038 return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(), 3039 ST.getMaxNumSGPRs(MF) / 4); 3040 } 3041 3042 ArrayRef<MCPhysReg> 3043 SIRegisterInfo::getAllSGPR64(const MachineFunction &MF) const { 3044 return makeArrayRef(AMDGPU::SGPR_64RegClass.begin(), 3045 ST.getMaxNumSGPRs(MF) / 2); 3046 } 3047 3048 ArrayRef<MCPhysReg> 3049 SIRegisterInfo::getAllSGPR32(const MachineFunction &MF) const { 3050 return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF)); 3051 } 3052