1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// \brief SI implementation of the TargetRegisterInfo class. 12 // 13 //===----------------------------------------------------------------------===// 14 15 16 #include "SIRegisterInfo.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "llvm/CodeGen/MachineFrameInfo.h" 20 #include "llvm/CodeGen/MachineInstrBuilder.h" 21 #include "llvm/CodeGen/RegisterScavenging.h" 22 #include "llvm/IR/Function.h" 23 #include "llvm/IR/LLVMContext.h" 24 25 using namespace llvm; 26 27 SIRegisterInfo::SIRegisterInfo() : AMDGPURegisterInfo() {} 28 29 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 30 BitVector Reserved(getNumRegs()); 31 Reserved.set(AMDGPU::EXEC); 32 33 // EXEC_LO and EXEC_HI could be allocated and used as regular register, 34 // but this seems likely to result in bugs, so I'm marking them as reserved. 35 Reserved.set(AMDGPU::EXEC_LO); 36 Reserved.set(AMDGPU::EXEC_HI); 37 38 Reserved.set(AMDGPU::INDIRECT_BASE_ADDR); 39 Reserved.set(AMDGPU::FLAT_SCR); 40 Reserved.set(AMDGPU::FLAT_SCR_LO); 41 Reserved.set(AMDGPU::FLAT_SCR_HI); 42 43 // Reserve some VGPRs to use as temp registers in case we have to spill VGPRs 44 Reserved.set(AMDGPU::VGPR255); 45 Reserved.set(AMDGPU::VGPR254); 46 47 // Tonga and Iceland can only allocate a fixed number of SGPRs due 48 // to a hw bug. 49 if (MF.getSubtarget<AMDGPUSubtarget>().hasSGPRInitBug()) { 50 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 51 // Reserve some SGPRs for FLAT_SCRATCH and VCC (4 SGPRs). 52 // Assume XNACK_MASK is unused. 53 unsigned Limit = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4; 54 55 for (unsigned i = Limit; i < NumSGPRs; ++i) { 56 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 57 MCRegAliasIterator R = MCRegAliasIterator(Reg, this, true); 58 59 for (; R.isValid(); ++R) 60 Reserved.set(*R); 61 } 62 } 63 64 return Reserved; 65 } 66 67 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 68 unsigned Idx) const { 69 70 const AMDGPUSubtarget &STI = MF.getSubtarget<AMDGPUSubtarget>(); 71 // FIXME: We should adjust the max number of waves based on LDS size. 72 unsigned SGPRLimit = getNumSGPRsAllowed(STI.getGeneration(), 73 STI.getMaxWavesPerCU()); 74 unsigned VGPRLimit = getNumVGPRsAllowed(STI.getMaxWavesPerCU()); 75 76 for (regclass_iterator I = regclass_begin(), E = regclass_end(); 77 I != E; ++I) { 78 79 unsigned NumSubRegs = std::max((int)(*I)->getSize() / 4, 1); 80 unsigned Limit; 81 82 if (isSGPRClass(*I)) { 83 Limit = SGPRLimit / NumSubRegs; 84 } else { 85 Limit = VGPRLimit / NumSubRegs; 86 } 87 88 const int *Sets = getRegClassPressureSets(*I); 89 assert(Sets); 90 for (unsigned i = 0; Sets[i] != -1; ++i) { 91 if (Sets[i] == (int)Idx) 92 return Limit; 93 } 94 } 95 return 256; 96 } 97 98 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 99 return Fn.getFrameInfo()->hasStackObjects(); 100 } 101 102 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 103 104 switch (Op) { 105 case AMDGPU::SI_SPILL_S512_SAVE: 106 case AMDGPU::SI_SPILL_S512_RESTORE: 107 case AMDGPU::SI_SPILL_V512_SAVE: 108 case AMDGPU::SI_SPILL_V512_RESTORE: 109 return 16; 110 case AMDGPU::SI_SPILL_S256_SAVE: 111 case AMDGPU::SI_SPILL_S256_RESTORE: 112 case AMDGPU::SI_SPILL_V256_SAVE: 113 case AMDGPU::SI_SPILL_V256_RESTORE: 114 return 8; 115 case AMDGPU::SI_SPILL_S128_SAVE: 116 case AMDGPU::SI_SPILL_S128_RESTORE: 117 case AMDGPU::SI_SPILL_V128_SAVE: 118 case AMDGPU::SI_SPILL_V128_RESTORE: 119 return 4; 120 case AMDGPU::SI_SPILL_V96_SAVE: 121 case AMDGPU::SI_SPILL_V96_RESTORE: 122 return 3; 123 case AMDGPU::SI_SPILL_S64_SAVE: 124 case AMDGPU::SI_SPILL_S64_RESTORE: 125 case AMDGPU::SI_SPILL_V64_SAVE: 126 case AMDGPU::SI_SPILL_V64_RESTORE: 127 return 2; 128 case AMDGPU::SI_SPILL_S32_SAVE: 129 case AMDGPU::SI_SPILL_S32_RESTORE: 130 case AMDGPU::SI_SPILL_V32_SAVE: 131 case AMDGPU::SI_SPILL_V32_RESTORE: 132 return 1; 133 default: llvm_unreachable("Invalid spill opcode"); 134 } 135 } 136 137 void SIRegisterInfo::buildScratchLoadStore(MachineBasicBlock::iterator MI, 138 unsigned LoadStoreOp, 139 unsigned Value, 140 unsigned ScratchRsrcReg, 141 unsigned ScratchOffset, 142 int64_t Offset, 143 RegScavenger *RS) const { 144 145 MachineBasicBlock *MBB = MI->getParent(); 146 const MachineFunction *MF = MI->getParent()->getParent(); 147 const SIInstrInfo *TII = 148 static_cast<const SIInstrInfo *>(MF->getSubtarget().getInstrInfo()); 149 LLVMContext &Ctx = MF->getFunction()->getContext(); 150 DebugLoc DL = MI->getDebugLoc(); 151 bool IsLoad = TII->get(LoadStoreOp).mayLoad(); 152 153 bool RanOutOfSGPRs = false; 154 unsigned SOffset = ScratchOffset; 155 156 unsigned NumSubRegs = getNumSubRegsForSpillOp(MI->getOpcode()); 157 unsigned Size = NumSubRegs * 4; 158 159 if (!isUInt<12>(Offset + Size)) { 160 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0); 161 if (SOffset == AMDGPU::NoRegister) { 162 RanOutOfSGPRs = true; 163 SOffset = AMDGPU::SGPR0; 164 } 165 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 166 .addReg(ScratchOffset) 167 .addImm(Offset); 168 Offset = 0; 169 } 170 171 if (RanOutOfSGPRs) 172 Ctx.emitError("Ran out of SGPRs for spilling VGPRS"); 173 174 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += 4) { 175 unsigned SubReg = NumSubRegs > 1 ? 176 getPhysRegSubReg(Value, &AMDGPU::VGPR_32RegClass, i) : 177 Value; 178 bool IsKill = (i == e - 1); 179 180 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 181 .addReg(SubReg, getDefRegState(IsLoad)) 182 .addReg(ScratchRsrcReg, getKillRegState(IsKill)) 183 .addReg(SOffset) 184 .addImm(Offset) 185 .addImm(0) // glc 186 .addImm(0) // slc 187 .addImm(0) // tfe 188 .addReg(Value, RegState::Implicit | getDefRegState(IsLoad)); 189 } 190 } 191 192 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 193 int SPAdj, unsigned FIOperandNum, 194 RegScavenger *RS) const { 195 MachineFunction *MF = MI->getParent()->getParent(); 196 MachineBasicBlock *MBB = MI->getParent(); 197 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 198 MachineFrameInfo *FrameInfo = MF->getFrameInfo(); 199 const SIInstrInfo *TII = 200 static_cast<const SIInstrInfo *>(MF->getSubtarget().getInstrInfo()); 201 DebugLoc DL = MI->getDebugLoc(); 202 203 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 204 int Index = MI->getOperand(FIOperandNum).getIndex(); 205 206 switch (MI->getOpcode()) { 207 // SGPR register spill 208 case AMDGPU::SI_SPILL_S512_SAVE: 209 case AMDGPU::SI_SPILL_S256_SAVE: 210 case AMDGPU::SI_SPILL_S128_SAVE: 211 case AMDGPU::SI_SPILL_S64_SAVE: 212 case AMDGPU::SI_SPILL_S32_SAVE: { 213 unsigned NumSubRegs = getNumSubRegsForSpillOp(MI->getOpcode()); 214 215 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 216 unsigned SubReg = getPhysRegSubReg(MI->getOperand(0).getReg(), 217 &AMDGPU::SGPR_32RegClass, i); 218 struct SIMachineFunctionInfo::SpilledReg Spill = 219 MFI->getSpilledReg(MF, Index, i); 220 221 if (Spill.VGPR == AMDGPU::NoRegister) { 222 LLVMContext &Ctx = MF->getFunction()->getContext(); 223 Ctx.emitError("Ran out of VGPRs for spilling SGPR"); 224 } 225 226 BuildMI(*MBB, MI, DL, 227 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 228 Spill.VGPR) 229 .addReg(SubReg) 230 .addImm(Spill.Lane); 231 232 } 233 MI->eraseFromParent(); 234 break; 235 } 236 237 // SGPR register restore 238 case AMDGPU::SI_SPILL_S512_RESTORE: 239 case AMDGPU::SI_SPILL_S256_RESTORE: 240 case AMDGPU::SI_SPILL_S128_RESTORE: 241 case AMDGPU::SI_SPILL_S64_RESTORE: 242 case AMDGPU::SI_SPILL_S32_RESTORE: { 243 unsigned NumSubRegs = getNumSubRegsForSpillOp(MI->getOpcode()); 244 245 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 246 unsigned SubReg = getPhysRegSubReg(MI->getOperand(0).getReg(), 247 &AMDGPU::SGPR_32RegClass, i); 248 struct SIMachineFunctionInfo::SpilledReg Spill = 249 MFI->getSpilledReg(MF, Index, i); 250 251 if (Spill.VGPR == AMDGPU::NoRegister) { 252 LLVMContext &Ctx = MF->getFunction()->getContext(); 253 Ctx.emitError("Ran out of VGPRs for spilling SGPR"); 254 } 255 256 BuildMI(*MBB, MI, DL, 257 TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 258 SubReg) 259 .addReg(Spill.VGPR) 260 .addImm(Spill.Lane) 261 .addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 262 } 263 264 // TODO: only do this when it is needed 265 switch (MF->getSubtarget<AMDGPUSubtarget>().getGeneration()) { 266 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 267 // "VALU writes SGPR" -> "SMRD reads that SGPR" needs "S_NOP 3" on SI 268 TII->insertNOPs(MI, 3); 269 break; 270 case AMDGPUSubtarget::SEA_ISLANDS: 271 break; 272 default: // VOLCANIC_ISLANDS and later 273 // "VALU writes SGPR -> VMEM reads that SGPR" needs "S_NOP 4" on VI 274 // and later. This also applies to VALUs which write VCC, but we're 275 // unlikely to see VMEM use VCC. 276 TII->insertNOPs(MI, 4); 277 } 278 279 MI->eraseFromParent(); 280 break; 281 } 282 283 // VGPR register spill 284 case AMDGPU::SI_SPILL_V512_SAVE: 285 case AMDGPU::SI_SPILL_V256_SAVE: 286 case AMDGPU::SI_SPILL_V128_SAVE: 287 case AMDGPU::SI_SPILL_V96_SAVE: 288 case AMDGPU::SI_SPILL_V64_SAVE: 289 case AMDGPU::SI_SPILL_V32_SAVE: 290 buildScratchLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 291 TII->getNamedOperand(*MI, AMDGPU::OpName::src)->getReg(), 292 TII->getNamedOperand(*MI, AMDGPU::OpName::scratch_rsrc)->getReg(), 293 TII->getNamedOperand(*MI, AMDGPU::OpName::scratch_offset)->getReg(), 294 FrameInfo->getObjectOffset(Index), RS); 295 MI->eraseFromParent(); 296 break; 297 case AMDGPU::SI_SPILL_V32_RESTORE: 298 case AMDGPU::SI_SPILL_V64_RESTORE: 299 case AMDGPU::SI_SPILL_V96_RESTORE: 300 case AMDGPU::SI_SPILL_V128_RESTORE: 301 case AMDGPU::SI_SPILL_V256_RESTORE: 302 case AMDGPU::SI_SPILL_V512_RESTORE: { 303 buildScratchLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 304 TII->getNamedOperand(*MI, AMDGPU::OpName::dst)->getReg(), 305 TII->getNamedOperand(*MI, AMDGPU::OpName::scratch_rsrc)->getReg(), 306 TII->getNamedOperand(*MI, AMDGPU::OpName::scratch_offset)->getReg(), 307 FrameInfo->getObjectOffset(Index), RS); 308 MI->eraseFromParent(); 309 break; 310 } 311 312 default: { 313 int64_t Offset = FrameInfo->getObjectOffset(Index); 314 FIOp.ChangeToImmediate(Offset); 315 if (!TII->isImmOperandLegal(MI, FIOperandNum, FIOp)) { 316 unsigned TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, SPAdj); 317 BuildMI(*MBB, MI, MI->getDebugLoc(), 318 TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 319 .addImm(Offset); 320 FIOp.ChangeToRegister(TmpReg, false, false, true); 321 } 322 } 323 } 324 } 325 326 const TargetRegisterClass * SIRegisterInfo::getCFGStructurizerRegClass( 327 MVT VT) const { 328 switch(VT.SimpleTy) { 329 default: 330 case MVT::i32: return &AMDGPU::VGPR_32RegClass; 331 } 332 } 333 334 unsigned SIRegisterInfo::getHWRegIndex(unsigned Reg) const { 335 return getEncodingValue(Reg) & 0xff; 336 } 337 338 const TargetRegisterClass *SIRegisterInfo::getPhysRegClass(unsigned Reg) const { 339 assert(!TargetRegisterInfo::isVirtualRegister(Reg)); 340 341 static const TargetRegisterClass *BaseClasses[] = { 342 &AMDGPU::VGPR_32RegClass, 343 &AMDGPU::SReg_32RegClass, 344 &AMDGPU::VReg_64RegClass, 345 &AMDGPU::SReg_64RegClass, 346 &AMDGPU::VReg_96RegClass, 347 &AMDGPU::VReg_128RegClass, 348 &AMDGPU::SReg_128RegClass, 349 &AMDGPU::VReg_256RegClass, 350 &AMDGPU::SReg_256RegClass, 351 &AMDGPU::VReg_512RegClass 352 }; 353 354 for (const TargetRegisterClass *BaseClass : BaseClasses) { 355 if (BaseClass->contains(Reg)) { 356 return BaseClass; 357 } 358 } 359 return nullptr; 360 } 361 362 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 363 return getCommonSubClass(&AMDGPU::VGPR_32RegClass, RC) || 364 getCommonSubClass(&AMDGPU::VReg_64RegClass, RC) || 365 getCommonSubClass(&AMDGPU::VReg_96RegClass, RC) || 366 getCommonSubClass(&AMDGPU::VReg_128RegClass, RC) || 367 getCommonSubClass(&AMDGPU::VReg_256RegClass, RC) || 368 getCommonSubClass(&AMDGPU::VReg_512RegClass, RC); 369 } 370 371 const TargetRegisterClass *SIRegisterInfo::getEquivalentVGPRClass( 372 const TargetRegisterClass *SRC) const { 373 if (hasVGPRs(SRC)) { 374 return SRC; 375 } else if (SRC == &AMDGPU::SCCRegRegClass) { 376 return &AMDGPU::VCCRegRegClass; 377 } else if (getCommonSubClass(SRC, &AMDGPU::SGPR_32RegClass)) { 378 return &AMDGPU::VGPR_32RegClass; 379 } else if (getCommonSubClass(SRC, &AMDGPU::SGPR_64RegClass)) { 380 return &AMDGPU::VReg_64RegClass; 381 } else if (getCommonSubClass(SRC, &AMDGPU::SReg_128RegClass)) { 382 return &AMDGPU::VReg_128RegClass; 383 } else if (getCommonSubClass(SRC, &AMDGPU::SReg_256RegClass)) { 384 return &AMDGPU::VReg_256RegClass; 385 } else if (getCommonSubClass(SRC, &AMDGPU::SReg_512RegClass)) { 386 return &AMDGPU::VReg_512RegClass; 387 } 388 return nullptr; 389 } 390 391 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 392 const TargetRegisterClass *RC, unsigned SubIdx) const { 393 if (SubIdx == AMDGPU::NoSubRegister) 394 return RC; 395 396 // If this register has a sub-register, we can safely assume it is a 32-bit 397 // register, because all of SI's sub-registers are 32-bit. 398 if (isSGPRClass(RC)) { 399 return &AMDGPU::SGPR_32RegClass; 400 } else { 401 return &AMDGPU::VGPR_32RegClass; 402 } 403 } 404 405 unsigned SIRegisterInfo::getPhysRegSubReg(unsigned Reg, 406 const TargetRegisterClass *SubRC, 407 unsigned Channel) const { 408 409 switch (Reg) { 410 case AMDGPU::VCC: 411 switch(Channel) { 412 case 0: return AMDGPU::VCC_LO; 413 case 1: return AMDGPU::VCC_HI; 414 default: llvm_unreachable("Invalid SubIdx for VCC"); 415 } 416 417 case AMDGPU::FLAT_SCR: 418 switch (Channel) { 419 case 0: 420 return AMDGPU::FLAT_SCR_LO; 421 case 1: 422 return AMDGPU::FLAT_SCR_HI; 423 default: 424 llvm_unreachable("Invalid SubIdx for FLAT_SCR"); 425 } 426 break; 427 428 case AMDGPU::EXEC: 429 switch (Channel) { 430 case 0: 431 return AMDGPU::EXEC_LO; 432 case 1: 433 return AMDGPU::EXEC_HI; 434 default: 435 llvm_unreachable("Invalid SubIdx for EXEC"); 436 } 437 break; 438 } 439 440 const TargetRegisterClass *RC = getPhysRegClass(Reg); 441 // 32-bit registers don't have sub-registers, so we can just return the 442 // Reg. We need to have this check here, because the calculation below 443 // using getHWRegIndex() will fail with special 32-bit registers like 444 // VCC_LO, VCC_HI, EXEC_LO, EXEC_HI and M0. 445 if (RC->getSize() == 4) { 446 assert(Channel == 0); 447 return Reg; 448 } 449 450 unsigned Index = getHWRegIndex(Reg); 451 return SubRC->getRegister(Index + Channel); 452 } 453 454 bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const { 455 return OpType == AMDGPU::OPERAND_REG_IMM32; 456 } 457 458 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 459 if (opCanUseLiteralConstant(OpType)) 460 return true; 461 462 return OpType == AMDGPU::OPERAND_REG_INLINE_C; 463 } 464 465 unsigned SIRegisterInfo::getPreloadedValue(const MachineFunction &MF, 466 enum PreloadedValue Value) const { 467 468 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 469 switch (Value) { 470 case SIRegisterInfo::TGID_X: 471 return AMDGPU::SReg_32RegClass.getRegister(MFI->NumUserSGPRs + 0); 472 case SIRegisterInfo::TGID_Y: 473 return AMDGPU::SReg_32RegClass.getRegister(MFI->NumUserSGPRs + 1); 474 case SIRegisterInfo::TGID_Z: 475 return AMDGPU::SReg_32RegClass.getRegister(MFI->NumUserSGPRs + 2); 476 case SIRegisterInfo::SCRATCH_WAVE_OFFSET: 477 if (MFI->getShaderType() != ShaderType::COMPUTE) 478 return MFI->ScratchOffsetReg; 479 return AMDGPU::SReg_32RegClass.getRegister(MFI->NumUserSGPRs + 4); 480 case SIRegisterInfo::SCRATCH_PTR: 481 return AMDGPU::SGPR2_SGPR3; 482 case SIRegisterInfo::INPUT_PTR: 483 return AMDGPU::SGPR0_SGPR1; 484 case SIRegisterInfo::TIDIG_X: 485 return AMDGPU::VGPR0; 486 case SIRegisterInfo::TIDIG_Y: 487 return AMDGPU::VGPR1; 488 case SIRegisterInfo::TIDIG_Z: 489 return AMDGPU::VGPR2; 490 } 491 llvm_unreachable("unexpected preloaded value type"); 492 } 493 494 /// \brief Returns a register that is not used at any point in the function. 495 /// If all registers are used, then this function will return 496 // AMDGPU::NoRegister. 497 unsigned SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 498 const TargetRegisterClass *RC) const { 499 500 for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); 501 I != E; ++I) { 502 if (MRI.reg_nodbg_empty(*I)) 503 return *I; 504 } 505 return AMDGPU::NoRegister; 506 } 507 508 unsigned SIRegisterInfo::getNumVGPRsAllowed(unsigned WaveCount) const { 509 switch(WaveCount) { 510 case 10: return 24; 511 case 9: return 28; 512 case 8: return 32; 513 case 7: return 36; 514 case 6: return 40; 515 case 5: return 48; 516 case 4: return 64; 517 case 3: return 84; 518 case 2: return 128; 519 default: return 256; 520 } 521 } 522 523 unsigned SIRegisterInfo::getNumSGPRsAllowed(AMDGPUSubtarget::Generation gen, 524 unsigned WaveCount) const { 525 if (gen >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 526 switch (WaveCount) { 527 case 10: return 80; 528 case 9: return 80; 529 case 8: return 96; 530 default: return 102; 531 } 532 } else { 533 switch(WaveCount) { 534 case 10: return 48; 535 case 9: return 56; 536 case 8: return 64; 537 case 7: return 72; 538 case 6: return 80; 539 case 5: return 96; 540 default: return 103; 541 } 542 } 543 } 544