1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "AMDGPUSubtarget.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "llvm/CodeGen/LiveIntervals.h" 21 #include "llvm/CodeGen/MachineDominators.h" 22 #include "llvm/CodeGen/MachineFrameInfo.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/RegisterScavenging.h" 25 #include "llvm/CodeGen/SlotIndexes.h" 26 #include "llvm/IR/Function.h" 27 #include "llvm/IR/LLVMContext.h" 28 29 using namespace llvm; 30 31 static bool hasPressureSet(const int *PSets, unsigned PSetID) { 32 for (unsigned i = 0; PSets[i] != -1; ++i) { 33 if (PSets[i] == (int)PSetID) 34 return true; 35 } 36 return false; 37 } 38 39 void SIRegisterInfo::classifyPressureSet(unsigned PSetID, unsigned Reg, 40 BitVector &PressureSets) const { 41 for (MCRegUnitIterator U(Reg, this); U.isValid(); ++U) { 42 const int *PSets = getRegUnitPressureSets(*U); 43 if (hasPressureSet(PSets, PSetID)) { 44 PressureSets.set(PSetID); 45 break; 46 } 47 } 48 } 49 50 static cl::opt<bool> EnableSpillSGPRToSMEM( 51 "amdgpu-spill-sgpr-to-smem", 52 cl::desc("Use scalar stores to spill SGPRs if supported by subtarget"), 53 cl::init(false)); 54 55 static cl::opt<bool> EnableSpillSGPRToVGPR( 56 "amdgpu-spill-sgpr-to-vgpr", 57 cl::desc("Enable spilling VGPRs to SGPRs"), 58 cl::ReallyHidden, 59 cl::init(true)); 60 61 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) : 62 AMDGPURegisterInfo(), 63 SGPRPressureSets(getNumRegPressureSets()), 64 VGPRPressureSets(getNumRegPressureSets()), 65 SpillSGPRToVGPR(false), 66 SpillSGPRToSMEM(false) { 67 if (EnableSpillSGPRToSMEM && ST.hasScalarStores()) 68 SpillSGPRToSMEM = true; 69 else if (EnableSpillSGPRToVGPR) 70 SpillSGPRToVGPR = true; 71 72 unsigned NumRegPressureSets = getNumRegPressureSets(); 73 74 SGPRSetID = NumRegPressureSets; 75 VGPRSetID = NumRegPressureSets; 76 77 for (unsigned i = 0; i < NumRegPressureSets; ++i) { 78 classifyPressureSet(i, AMDGPU::SGPR0, SGPRPressureSets); 79 classifyPressureSet(i, AMDGPU::VGPR0, VGPRPressureSets); 80 } 81 82 // Determine the number of reg units for each pressure set. 83 std::vector<unsigned> PressureSetRegUnits(NumRegPressureSets, 0); 84 for (unsigned i = 0, e = getNumRegUnits(); i != e; ++i) { 85 const int *PSets = getRegUnitPressureSets(i); 86 for (unsigned j = 0; PSets[j] != -1; ++j) { 87 ++PressureSetRegUnits[PSets[j]]; 88 } 89 } 90 91 unsigned VGPRMax = 0, SGPRMax = 0; 92 for (unsigned i = 0; i < NumRegPressureSets; ++i) { 93 if (isVGPRPressureSet(i) && PressureSetRegUnits[i] > VGPRMax) { 94 VGPRSetID = i; 95 VGPRMax = PressureSetRegUnits[i]; 96 continue; 97 } 98 if (isSGPRPressureSet(i) && PressureSetRegUnits[i] > SGPRMax) { 99 SGPRSetID = i; 100 SGPRMax = PressureSetRegUnits[i]; 101 } 102 } 103 104 assert(SGPRSetID < NumRegPressureSets && 105 VGPRSetID < NumRegPressureSets); 106 } 107 108 unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg( 109 const MachineFunction &MF) const { 110 111 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 112 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 113 unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 114 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass); 115 } 116 117 static unsigned findPrivateSegmentWaveByteOffsetRegIndex(unsigned RegCount) { 118 unsigned Reg; 119 120 // Try to place it in a hole after PrivateSegmentBufferReg. 121 if (RegCount & 3) { 122 // We cannot put the segment buffer in (Idx - 4) ... (Idx - 1) due to 123 // alignment constraints, so we have a hole where can put the wave offset. 124 Reg = RegCount - 1; 125 } else { 126 // We can put the segment buffer in (Idx - 4) ... (Idx - 1) and put the 127 // wave offset before it. 128 Reg = RegCount - 5; 129 } 130 131 return Reg; 132 } 133 134 unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg( 135 const MachineFunction &MF) const { 136 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 137 unsigned Reg = findPrivateSegmentWaveByteOffsetRegIndex(ST.getMaxNumSGPRs(MF)); 138 return AMDGPU::SGPR_32RegClass.getRegister(Reg); 139 } 140 141 unsigned SIRegisterInfo::reservedStackPtrOffsetReg( 142 const MachineFunction &MF) const { 143 return AMDGPU::SGPR32; 144 } 145 146 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 147 BitVector Reserved(getNumRegs()); 148 149 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 150 // this seems likely to result in bugs, so I'm marking them as reserved. 151 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 152 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 153 154 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 155 reserveRegisterTuples(Reserved, AMDGPU::M0); 156 157 // Reserve the memory aperture registers. 158 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 159 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 160 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 161 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 162 163 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 164 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 165 166 // Reserve xnack_mask registers - support is not implemented in Codegen. 167 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 168 169 // Reserve lds_direct register - support is not implemented in Codegen. 170 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 171 172 // Reserve Trap Handler registers - support is not implemented in Codegen. 173 reserveRegisterTuples(Reserved, AMDGPU::TBA); 174 reserveRegisterTuples(Reserved, AMDGPU::TMA); 175 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 176 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 177 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 178 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 179 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 180 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 181 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 182 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 183 184 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 185 186 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 187 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 188 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 189 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 190 reserveRegisterTuples(Reserved, Reg); 191 } 192 193 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 194 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 195 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 196 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 197 reserveRegisterTuples(Reserved, Reg); 198 } 199 200 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 201 202 unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); 203 if (ScratchWaveOffsetReg != AMDGPU::NoRegister) { 204 // Reserve 1 SGPR for scratch wave offset in case we need to spill. 205 reserveRegisterTuples(Reserved, ScratchWaveOffsetReg); 206 } 207 208 unsigned ScratchRSrcReg = MFI->getScratchRSrcReg(); 209 if (ScratchRSrcReg != AMDGPU::NoRegister) { 210 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 211 // to spill. 212 // TODO: May need to reserve a VGPR if doing LDS spilling. 213 reserveRegisterTuples(Reserved, ScratchRSrcReg); 214 assert(!isSubRegister(ScratchRSrcReg, ScratchWaveOffsetReg)); 215 } 216 217 // We have to assume the SP is needed in case there are calls in the function, 218 // which is detected after the function is lowered. If we aren't really going 219 // to need SP, don't bother reserving it. 220 unsigned StackPtrReg = MFI->getStackPtrOffsetReg(); 221 222 if (StackPtrReg != AMDGPU::NoRegister) { 223 reserveRegisterTuples(Reserved, StackPtrReg); 224 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 225 } 226 227 unsigned FrameReg = MFI->getFrameOffsetReg(); 228 if (FrameReg != AMDGPU::NoRegister) { 229 reserveRegisterTuples(Reserved, FrameReg); 230 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 231 } 232 233 return Reserved; 234 } 235 236 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 237 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 238 if (Info->isEntryFunction()) { 239 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 240 return MFI.hasStackObjects() || MFI.hasCalls(); 241 } 242 243 // May need scavenger for dealing with callee saved registers. 244 return true; 245 } 246 247 bool SIRegisterInfo::requiresFrameIndexScavenging( 248 const MachineFunction &MF) const { 249 const MachineFrameInfo &MFI = MF.getFrameInfo(); 250 if (MFI.hasStackObjects()) 251 return true; 252 253 // May need to deal with callee saved registers. 254 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 255 return !Info->isEntryFunction(); 256 } 257 258 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 259 const MachineFunction &MF) const { 260 // m0 is needed for the scalar store offset. m0 is unallocatable, so we can't 261 // create a virtual register for it during frame index elimination, so the 262 // scavenger is directly needed. 263 return MF.getFrameInfo().hasStackObjects() && 264 MF.getSubtarget<GCNSubtarget>().hasScalarStores() && 265 MF.getInfo<SIMachineFunctionInfo>()->hasSpilledSGPRs(); 266 } 267 268 bool SIRegisterInfo::requiresVirtualBaseRegisters( 269 const MachineFunction &) const { 270 // There are no special dedicated stack or frame pointers. 271 return true; 272 } 273 274 bool SIRegisterInfo::trackLivenessAfterRegAlloc(const MachineFunction &MF) const { 275 // This helps catch bugs as verifier errors. 276 return true; 277 } 278 279 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const { 280 assert(SIInstrInfo::isMUBUF(*MI)); 281 282 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 283 AMDGPU::OpName::offset); 284 return MI->getOperand(OffIdx).getImm(); 285 } 286 287 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 288 int Idx) const { 289 if (!SIInstrInfo::isMUBUF(*MI)) 290 return 0; 291 292 assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 293 AMDGPU::OpName::vaddr) && 294 "Should never see frame index on non-address operand"); 295 296 return getMUBUFInstrOffset(MI); 297 } 298 299 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 300 if (!MI->mayLoadOrStore()) 301 return false; 302 303 int64_t FullOffset = Offset + getMUBUFInstrOffset(MI); 304 305 return !isUInt<12>(FullOffset); 306 } 307 308 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 309 unsigned BaseReg, 310 int FrameIdx, 311 int64_t Offset) const { 312 MachineBasicBlock::iterator Ins = MBB->begin(); 313 DebugLoc DL; // Defaults to "unknown" 314 315 if (Ins != MBB->end()) 316 DL = Ins->getDebugLoc(); 317 318 MachineFunction *MF = MBB->getParent(); 319 const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>(); 320 const SIInstrInfo *TII = Subtarget.getInstrInfo(); 321 322 if (Offset == 0) { 323 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) 324 .addFrameIndex(FrameIdx); 325 return; 326 } 327 328 MachineRegisterInfo &MRI = MF->getRegInfo(); 329 unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 330 331 unsigned FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 332 333 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 334 .addImm(Offset); 335 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg) 336 .addFrameIndex(FrameIdx); 337 338 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 339 .addReg(OffsetReg, RegState::Kill) 340 .addReg(FIReg) 341 .addImm(0); // clamp bit 342 } 343 344 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg, 345 int64_t Offset) const { 346 347 MachineBasicBlock *MBB = MI.getParent(); 348 MachineFunction *MF = MBB->getParent(); 349 const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>(); 350 const SIInstrInfo *TII = Subtarget.getInstrInfo(); 351 352 #ifndef NDEBUG 353 // FIXME: Is it possible to be storing a frame index to itself? 354 bool SeenFI = false; 355 for (const MachineOperand &MO: MI.operands()) { 356 if (MO.isFI()) { 357 if (SeenFI) 358 llvm_unreachable("should not see multiple frame indices"); 359 360 SeenFI = true; 361 } 362 } 363 #endif 364 365 MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 366 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 367 assert(TII->isMUBUF(MI)); 368 assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() == 369 MF->getInfo<SIMachineFunctionInfo>()->getFrameOffsetReg() && 370 "should only be seeing frame offset relative FrameIndex"); 371 372 373 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 374 int64_t NewOffset = OffsetOp->getImm() + Offset; 375 assert(isUInt<12>(NewOffset) && "offset should be legal"); 376 377 FIOp->ChangeToRegister(BaseReg, false); 378 OffsetOp->setImm(NewOffset); 379 } 380 381 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 382 unsigned BaseReg, 383 int64_t Offset) const { 384 if (!SIInstrInfo::isMUBUF(*MI)) 385 return false; 386 387 int64_t NewOffset = Offset + getMUBUFInstrOffset(MI); 388 389 return isUInt<12>(NewOffset); 390 } 391 392 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 393 const MachineFunction &MF, unsigned Kind) const { 394 // This is inaccurate. It depends on the instruction and address space. The 395 // only place where we should hit this is for dealing with frame indexes / 396 // private accesses, so this is correct in that case. 397 return &AMDGPU::VGPR_32RegClass; 398 } 399 400 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 401 402 switch (Op) { 403 case AMDGPU::SI_SPILL_S512_SAVE: 404 case AMDGPU::SI_SPILL_S512_RESTORE: 405 case AMDGPU::SI_SPILL_V512_SAVE: 406 case AMDGPU::SI_SPILL_V512_RESTORE: 407 return 16; 408 case AMDGPU::SI_SPILL_S256_SAVE: 409 case AMDGPU::SI_SPILL_S256_RESTORE: 410 case AMDGPU::SI_SPILL_V256_SAVE: 411 case AMDGPU::SI_SPILL_V256_RESTORE: 412 return 8; 413 case AMDGPU::SI_SPILL_S160_SAVE: 414 case AMDGPU::SI_SPILL_S160_RESTORE: 415 case AMDGPU::SI_SPILL_V160_SAVE: 416 case AMDGPU::SI_SPILL_V160_RESTORE: 417 return 5; 418 case AMDGPU::SI_SPILL_S128_SAVE: 419 case AMDGPU::SI_SPILL_S128_RESTORE: 420 case AMDGPU::SI_SPILL_V128_SAVE: 421 case AMDGPU::SI_SPILL_V128_RESTORE: 422 return 4; 423 case AMDGPU::SI_SPILL_S96_SAVE: 424 case AMDGPU::SI_SPILL_S96_RESTORE: 425 case AMDGPU::SI_SPILL_V96_SAVE: 426 case AMDGPU::SI_SPILL_V96_RESTORE: 427 return 3; 428 case AMDGPU::SI_SPILL_S64_SAVE: 429 case AMDGPU::SI_SPILL_S64_RESTORE: 430 case AMDGPU::SI_SPILL_V64_SAVE: 431 case AMDGPU::SI_SPILL_V64_RESTORE: 432 return 2; 433 case AMDGPU::SI_SPILL_S32_SAVE: 434 case AMDGPU::SI_SPILL_S32_RESTORE: 435 case AMDGPU::SI_SPILL_V32_SAVE: 436 case AMDGPU::SI_SPILL_V32_RESTORE: 437 return 1; 438 default: llvm_unreachable("Invalid spill opcode"); 439 } 440 } 441 442 static int getOffsetMUBUFStore(unsigned Opc) { 443 switch (Opc) { 444 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 445 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 446 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 447 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 448 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 449 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 450 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 451 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 452 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 453 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 454 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 455 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 456 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 457 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 458 default: 459 return -1; 460 } 461 } 462 463 static int getOffsetMUBUFLoad(unsigned Opc) { 464 switch (Opc) { 465 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 466 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 467 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 468 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 469 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 470 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 471 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 472 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 473 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 474 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 475 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 476 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 477 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 478 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 479 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 480 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 481 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 482 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 483 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 484 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 485 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 486 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 487 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 488 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 489 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 490 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 491 default: 492 return -1; 493 } 494 } 495 496 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 497 // need to handle the case where an SGPR may need to be spilled while spilling. 498 static bool buildMUBUFOffsetLoadStore(const SIInstrInfo *TII, 499 MachineFrameInfo &MFI, 500 MachineBasicBlock::iterator MI, 501 int Index, 502 int64_t Offset) { 503 MachineBasicBlock *MBB = MI->getParent(); 504 const DebugLoc &DL = MI->getDebugLoc(); 505 bool IsStore = MI->mayStore(); 506 507 unsigned Opc = MI->getOpcode(); 508 int LoadStoreOp = IsStore ? 509 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 510 if (LoadStoreOp == -1) 511 return false; 512 513 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 514 MachineInstrBuilder NewMI = 515 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 516 .add(*Reg) 517 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 518 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 519 .addImm(Offset) 520 .addImm(0) // glc 521 .addImm(0) // slc 522 .addImm(0) // tfe 523 .cloneMemRefs(*MI); 524 525 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 526 AMDGPU::OpName::vdata_in); 527 if (VDataIn) 528 NewMI.add(*VDataIn); 529 return true; 530 } 531 532 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI, 533 unsigned LoadStoreOp, 534 int Index, 535 unsigned ValueReg, 536 bool IsKill, 537 unsigned ScratchRsrcReg, 538 unsigned ScratchOffsetReg, 539 int64_t InstOffset, 540 MachineMemOperand *MMO, 541 RegScavenger *RS) const { 542 MachineBasicBlock *MBB = MI->getParent(); 543 MachineFunction *MF = MI->getParent()->getParent(); 544 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 545 const SIInstrInfo *TII = ST.getInstrInfo(); 546 const MachineFrameInfo &MFI = MF->getFrameInfo(); 547 548 const MCInstrDesc &Desc = TII->get(LoadStoreOp); 549 const DebugLoc &DL = MI->getDebugLoc(); 550 bool IsStore = Desc.mayStore(); 551 552 bool Scavenged = false; 553 unsigned SOffset = ScratchOffsetReg; 554 555 const unsigned EltSize = 4; 556 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 557 unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT); 558 unsigned Size = NumSubRegs * EltSize; 559 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 560 int64_t ScratchOffsetRegDelta = 0; 561 562 unsigned Align = MFI.getObjectAlignment(Index); 563 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 564 565 assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset"); 566 567 if (!isUInt<12>(Offset + Size - EltSize)) { 568 SOffset = AMDGPU::NoRegister; 569 570 // We currently only support spilling VGPRs to EltSize boundaries, meaning 571 // we can simplify the adjustment of Offset here to just scale with 572 // WavefrontSize. 573 Offset *= ST.getWavefrontSize(); 574 575 // We don't have access to the register scavenger if this function is called 576 // during PEI::scavengeFrameVirtualRegs(). 577 if (RS) 578 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0, false); 579 580 if (SOffset == AMDGPU::NoRegister) { 581 // There are no free SGPRs, and since we are in the process of spilling 582 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 583 // on SI/CI and on VI it is true until we implement spilling using scalar 584 // stores), we have no way to free up an SGPR. Our solution here is to 585 // add the offset directly to the ScratchOffset register, and then 586 // subtract the offset after the spill to return ScratchOffset to it's 587 // original value. 588 SOffset = ScratchOffsetReg; 589 ScratchOffsetRegDelta = Offset; 590 } else { 591 Scavenged = true; 592 } 593 594 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 595 .addReg(ScratchOffsetReg) 596 .addImm(Offset); 597 598 Offset = 0; 599 } 600 601 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) { 602 unsigned SubReg = NumSubRegs == 1 ? 603 ValueReg : getSubReg(ValueReg, getSubRegFromChannel(i)); 604 605 unsigned SOffsetRegState = 0; 606 unsigned SrcDstRegState = getDefRegState(!IsStore); 607 if (i + 1 == e) { 608 SOffsetRegState |= getKillRegState(Scavenged); 609 // The last implicit use carries the "Kill" flag. 610 SrcDstRegState |= getKillRegState(IsKill); 611 } 612 613 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i); 614 MachineMemOperand *NewMMO 615 = MF->getMachineMemOperand(PInfo, MMO->getFlags(), 616 EltSize, MinAlign(Align, EltSize * i)); 617 618 auto MIB = BuildMI(*MBB, MI, DL, Desc) 619 .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill)) 620 .addReg(ScratchRsrcReg) 621 .addReg(SOffset, SOffsetRegState) 622 .addImm(Offset) 623 .addImm(0) // glc 624 .addImm(0) // slc 625 .addImm(0) // tfe 626 .addMemOperand(NewMMO); 627 628 if (NumSubRegs > 1) 629 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 630 } 631 632 if (ScratchOffsetRegDelta != 0) { 633 // Subtract the offset we added to the ScratchOffset register. 634 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScratchOffsetReg) 635 .addReg(ScratchOffsetReg) 636 .addImm(ScratchOffsetRegDelta); 637 } 638 } 639 640 static std::pair<unsigned, unsigned> getSpillEltSize(unsigned SuperRegSize, 641 bool Store) { 642 if (SuperRegSize % 16 == 0) { 643 return { 16, Store ? AMDGPU::S_BUFFER_STORE_DWORDX4_SGPR : 644 AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR }; 645 } 646 647 if (SuperRegSize % 8 == 0) { 648 return { 8, Store ? AMDGPU::S_BUFFER_STORE_DWORDX2_SGPR : 649 AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR }; 650 } 651 652 return { 4, Store ? AMDGPU::S_BUFFER_STORE_DWORD_SGPR : 653 AMDGPU::S_BUFFER_LOAD_DWORD_SGPR}; 654 } 655 656 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 657 int Index, 658 RegScavenger *RS, 659 bool OnlyToVGPR) const { 660 MachineBasicBlock *MBB = MI->getParent(); 661 MachineFunction *MF = MBB->getParent(); 662 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 663 DenseSet<unsigned> SGPRSpillVGPRDefinedSet; 664 665 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 666 = MFI->getSGPRToVGPRSpills(Index); 667 bool SpillToVGPR = !VGPRSpills.empty(); 668 if (OnlyToVGPR && !SpillToVGPR) 669 return false; 670 671 MachineRegisterInfo &MRI = MF->getRegInfo(); 672 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 673 const SIInstrInfo *TII = ST.getInstrInfo(); 674 675 unsigned SuperReg = MI->getOperand(0).getReg(); 676 bool IsKill = MI->getOperand(0).isKill(); 677 const DebugLoc &DL = MI->getDebugLoc(); 678 679 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 680 681 bool SpillToSMEM = spillSGPRToSMEM(); 682 if (SpillToSMEM && OnlyToVGPR) 683 return false; 684 685 assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() && 686 SuperReg != MFI->getFrameOffsetReg() && 687 SuperReg != MFI->getScratchWaveOffsetReg())); 688 689 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 690 691 unsigned OffsetReg = AMDGPU::M0; 692 unsigned M0CopyReg = AMDGPU::NoRegister; 693 694 if (SpillToSMEM) { 695 if (RS->isRegUsed(AMDGPU::M0)) { 696 M0CopyReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 697 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), M0CopyReg) 698 .addReg(AMDGPU::M0); 699 } 700 } 701 702 unsigned ScalarStoreOp; 703 unsigned EltSize = 4; 704 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 705 if (SpillToSMEM && isSGPRClass(RC)) { 706 // XXX - if private_element_size is larger than 4 it might be useful to be 707 // able to spill wider vmem spills. 708 std::tie(EltSize, ScalarStoreOp) = 709 getSpillEltSize(getRegSizeInBits(*RC) / 8, true); 710 } 711 712 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 713 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 714 715 // SubReg carries the "Kill" flag when SubReg == SuperReg. 716 unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill); 717 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 718 unsigned SubReg = NumSubRegs == 1 ? 719 SuperReg : getSubReg(SuperReg, SplitParts[i]); 720 721 if (SpillToSMEM) { 722 int64_t FrOffset = FrameInfo.getObjectOffset(Index); 723 724 // The allocated memory size is really the wavefront size * the frame 725 // index size. The widest register class is 64 bytes, so a 4-byte scratch 726 // allocation is enough to spill this in a single stack object. 727 // 728 // FIXME: Frame size/offsets are computed earlier than this, so the extra 729 // space is still unnecessarily allocated. 730 731 unsigned Align = FrameInfo.getObjectAlignment(Index); 732 MachinePointerInfo PtrInfo 733 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 734 MachineMemOperand *MMO 735 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 736 EltSize, MinAlign(Align, EltSize * i)); 737 738 // SMEM instructions only support a single offset, so increment the wave 739 // offset. 740 741 int64_t Offset = (ST.getWavefrontSize() * FrOffset) + (EltSize * i); 742 if (Offset != 0) { 743 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), OffsetReg) 744 .addReg(MFI->getFrameOffsetReg()) 745 .addImm(Offset); 746 } else { 747 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 748 .addReg(MFI->getFrameOffsetReg()); 749 } 750 751 BuildMI(*MBB, MI, DL, TII->get(ScalarStoreOp)) 752 .addReg(SubReg, getKillRegState(IsKill)) // sdata 753 .addReg(MFI->getScratchRSrcReg()) // sbase 754 .addReg(OffsetReg, RegState::Kill) // soff 755 .addImm(0) // glc 756 .addMemOperand(MMO); 757 758 continue; 759 } 760 761 if (SpillToVGPR) { 762 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 763 764 // During SGPR spilling to VGPR, determine if the VGPR is defined. The 765 // only circumstance in which we say it is undefined is when it is the 766 // first spill to this VGPR in the first basic block. 767 bool VGPRDefined = true; 768 if (MBB == &MF->front()) 769 VGPRDefined = !SGPRSpillVGPRDefinedSet.insert(Spill.VGPR).second; 770 771 // Mark the "old value of vgpr" input undef only if this is the first sgpr 772 // spill to this specific vgpr in the first basic block. 773 BuildMI(*MBB, MI, DL, 774 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 775 Spill.VGPR) 776 .addReg(SubReg, getKillRegState(IsKill)) 777 .addImm(Spill.Lane) 778 .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef); 779 780 // FIXME: Since this spills to another register instead of an actual 781 // frame index, we should delete the frame index when all references to 782 // it are fixed. 783 } else { 784 // XXX - Can to VGPR spill fail for some subregisters but not others? 785 if (OnlyToVGPR) 786 return false; 787 788 // Spill SGPR to a frame index. 789 // TODO: Should VI try to spill to VGPR and then spill to SMEM? 790 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 791 // TODO: Should VI try to spill to VGPR and then spill to SMEM? 792 793 MachineInstrBuilder Mov 794 = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 795 .addReg(SubReg, SubKillState); 796 797 798 // There could be undef components of a spilled super register. 799 // TODO: Can we detect this and skip the spill? 800 if (NumSubRegs > 1) { 801 // The last implicit use of the SuperReg carries the "Kill" flag. 802 unsigned SuperKillState = 0; 803 if (i + 1 == e) 804 SuperKillState |= getKillRegState(IsKill); 805 Mov.addReg(SuperReg, RegState::Implicit | SuperKillState); 806 } 807 808 unsigned Align = FrameInfo.getObjectAlignment(Index); 809 MachinePointerInfo PtrInfo 810 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 811 MachineMemOperand *MMO 812 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 813 EltSize, MinAlign(Align, EltSize * i)); 814 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE)) 815 .addReg(TmpReg, RegState::Kill) // src 816 .addFrameIndex(Index) // vaddr 817 .addReg(MFI->getScratchRSrcReg()) // srrsrc 818 .addReg(MFI->getFrameOffsetReg()) // soffset 819 .addImm(i * 4) // offset 820 .addMemOperand(MMO); 821 } 822 } 823 824 if (M0CopyReg != AMDGPU::NoRegister) { 825 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) 826 .addReg(M0CopyReg, RegState::Kill); 827 } 828 829 MI->eraseFromParent(); 830 MFI->addToSpilledSGPRs(NumSubRegs); 831 return true; 832 } 833 834 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 835 int Index, 836 RegScavenger *RS, 837 bool OnlyToVGPR) const { 838 MachineFunction *MF = MI->getParent()->getParent(); 839 MachineRegisterInfo &MRI = MF->getRegInfo(); 840 MachineBasicBlock *MBB = MI->getParent(); 841 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 842 843 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 844 = MFI->getSGPRToVGPRSpills(Index); 845 bool SpillToVGPR = !VGPRSpills.empty(); 846 if (OnlyToVGPR && !SpillToVGPR) 847 return false; 848 849 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 850 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 851 const SIInstrInfo *TII = ST.getInstrInfo(); 852 const DebugLoc &DL = MI->getDebugLoc(); 853 854 unsigned SuperReg = MI->getOperand(0).getReg(); 855 bool SpillToSMEM = spillSGPRToSMEM(); 856 if (SpillToSMEM && OnlyToVGPR) 857 return false; 858 859 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 860 861 unsigned OffsetReg = AMDGPU::M0; 862 unsigned M0CopyReg = AMDGPU::NoRegister; 863 864 if (SpillToSMEM) { 865 if (RS->isRegUsed(AMDGPU::M0)) { 866 M0CopyReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 867 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), M0CopyReg) 868 .addReg(AMDGPU::M0); 869 } 870 } 871 872 unsigned EltSize = 4; 873 unsigned ScalarLoadOp; 874 875 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 876 if (SpillToSMEM && isSGPRClass(RC)) { 877 // XXX - if private_element_size is larger than 4 it might be useful to be 878 // able to spill wider vmem spills. 879 std::tie(EltSize, ScalarLoadOp) = 880 getSpillEltSize(getRegSizeInBits(*RC) / 8, false); 881 } 882 883 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 884 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 885 886 // SubReg carries the "Kill" flag when SubReg == SuperReg. 887 int64_t FrOffset = FrameInfo.getObjectOffset(Index); 888 889 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 890 unsigned SubReg = NumSubRegs == 1 ? 891 SuperReg : getSubReg(SuperReg, SplitParts[i]); 892 893 if (SpillToSMEM) { 894 // FIXME: Size may be > 4 but extra bytes wasted. 895 unsigned Align = FrameInfo.getObjectAlignment(Index); 896 MachinePointerInfo PtrInfo 897 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 898 MachineMemOperand *MMO 899 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 900 EltSize, MinAlign(Align, EltSize * i)); 901 902 // Add i * 4 offset 903 int64_t Offset = (ST.getWavefrontSize() * FrOffset) + (EltSize * i); 904 if (Offset != 0) { 905 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), OffsetReg) 906 .addReg(MFI->getFrameOffsetReg()) 907 .addImm(Offset); 908 } else { 909 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 910 .addReg(MFI->getFrameOffsetReg()); 911 } 912 913 auto MIB = 914 BuildMI(*MBB, MI, DL, TII->get(ScalarLoadOp), SubReg) 915 .addReg(MFI->getScratchRSrcReg()) // sbase 916 .addReg(OffsetReg, RegState::Kill) // soff 917 .addImm(0) // glc 918 .addMemOperand(MMO); 919 920 if (NumSubRegs > 1 && i == 0) 921 MIB.addReg(SuperReg, RegState::ImplicitDefine); 922 923 continue; 924 } 925 926 if (SpillToVGPR) { 927 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 928 auto MIB = 929 BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 930 SubReg) 931 .addReg(Spill.VGPR) 932 .addImm(Spill.Lane); 933 934 if (NumSubRegs > 1 && i == 0) 935 MIB.addReg(SuperReg, RegState::ImplicitDefine); 936 } else { 937 if (OnlyToVGPR) 938 return false; 939 940 // Restore SGPR from a stack slot. 941 // FIXME: We should use S_LOAD_DWORD here for VI. 942 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 943 unsigned Align = FrameInfo.getObjectAlignment(Index); 944 945 MachinePointerInfo PtrInfo 946 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 947 948 MachineMemOperand *MMO = MF->getMachineMemOperand(PtrInfo, 949 MachineMemOperand::MOLoad, EltSize, 950 MinAlign(Align, EltSize * i)); 951 952 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpReg) 953 .addFrameIndex(Index) // vaddr 954 .addReg(MFI->getScratchRSrcReg()) // srsrc 955 .addReg(MFI->getFrameOffsetReg()) // soffset 956 .addImm(i * 4) // offset 957 .addMemOperand(MMO); 958 959 auto MIB = 960 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) 961 .addReg(TmpReg, RegState::Kill); 962 963 if (NumSubRegs > 1) 964 MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 965 } 966 } 967 968 if (M0CopyReg != AMDGPU::NoRegister) { 969 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) 970 .addReg(M0CopyReg, RegState::Kill); 971 } 972 973 MI->eraseFromParent(); 974 return true; 975 } 976 977 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 978 /// a VGPR and the stack slot can be safely eliminated when all other users are 979 /// handled. 980 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 981 MachineBasicBlock::iterator MI, 982 int FI, 983 RegScavenger *RS) const { 984 switch (MI->getOpcode()) { 985 case AMDGPU::SI_SPILL_S512_SAVE: 986 case AMDGPU::SI_SPILL_S256_SAVE: 987 case AMDGPU::SI_SPILL_S160_SAVE: 988 case AMDGPU::SI_SPILL_S128_SAVE: 989 case AMDGPU::SI_SPILL_S96_SAVE: 990 case AMDGPU::SI_SPILL_S64_SAVE: 991 case AMDGPU::SI_SPILL_S32_SAVE: 992 return spillSGPR(MI, FI, RS, true); 993 case AMDGPU::SI_SPILL_S512_RESTORE: 994 case AMDGPU::SI_SPILL_S256_RESTORE: 995 case AMDGPU::SI_SPILL_S160_RESTORE: 996 case AMDGPU::SI_SPILL_S128_RESTORE: 997 case AMDGPU::SI_SPILL_S96_RESTORE: 998 case AMDGPU::SI_SPILL_S64_RESTORE: 999 case AMDGPU::SI_SPILL_S32_RESTORE: 1000 return restoreSGPR(MI, FI, RS, true); 1001 default: 1002 llvm_unreachable("not an SGPR spill instruction"); 1003 } 1004 } 1005 1006 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1007 int SPAdj, unsigned FIOperandNum, 1008 RegScavenger *RS) const { 1009 MachineFunction *MF = MI->getParent()->getParent(); 1010 MachineRegisterInfo &MRI = MF->getRegInfo(); 1011 MachineBasicBlock *MBB = MI->getParent(); 1012 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1013 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1014 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 1015 const SIInstrInfo *TII = ST.getInstrInfo(); 1016 DebugLoc DL = MI->getDebugLoc(); 1017 1018 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1019 int Index = MI->getOperand(FIOperandNum).getIndex(); 1020 1021 switch (MI->getOpcode()) { 1022 // SGPR register spill 1023 case AMDGPU::SI_SPILL_S512_SAVE: 1024 case AMDGPU::SI_SPILL_S256_SAVE: 1025 case AMDGPU::SI_SPILL_S160_SAVE: 1026 case AMDGPU::SI_SPILL_S128_SAVE: 1027 case AMDGPU::SI_SPILL_S96_SAVE: 1028 case AMDGPU::SI_SPILL_S64_SAVE: 1029 case AMDGPU::SI_SPILL_S32_SAVE: { 1030 spillSGPR(MI, Index, RS); 1031 break; 1032 } 1033 1034 // SGPR register restore 1035 case AMDGPU::SI_SPILL_S512_RESTORE: 1036 case AMDGPU::SI_SPILL_S256_RESTORE: 1037 case AMDGPU::SI_SPILL_S160_RESTORE: 1038 case AMDGPU::SI_SPILL_S128_RESTORE: 1039 case AMDGPU::SI_SPILL_S96_RESTORE: 1040 case AMDGPU::SI_SPILL_S64_RESTORE: 1041 case AMDGPU::SI_SPILL_S32_RESTORE: { 1042 restoreSGPR(MI, Index, RS); 1043 break; 1044 } 1045 1046 // VGPR register spill 1047 case AMDGPU::SI_SPILL_V512_SAVE: 1048 case AMDGPU::SI_SPILL_V256_SAVE: 1049 case AMDGPU::SI_SPILL_V160_SAVE: 1050 case AMDGPU::SI_SPILL_V128_SAVE: 1051 case AMDGPU::SI_SPILL_V96_SAVE: 1052 case AMDGPU::SI_SPILL_V64_SAVE: 1053 case AMDGPU::SI_SPILL_V32_SAVE: { 1054 const MachineOperand *VData = TII->getNamedOperand(*MI, 1055 AMDGPU::OpName::vdata); 1056 buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 1057 Index, 1058 VData->getReg(), VData->isKill(), 1059 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1060 TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg(), 1061 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1062 *MI->memoperands_begin(), 1063 RS); 1064 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 1065 MI->eraseFromParent(); 1066 break; 1067 } 1068 case AMDGPU::SI_SPILL_V32_RESTORE: 1069 case AMDGPU::SI_SPILL_V64_RESTORE: 1070 case AMDGPU::SI_SPILL_V96_RESTORE: 1071 case AMDGPU::SI_SPILL_V128_RESTORE: 1072 case AMDGPU::SI_SPILL_V160_RESTORE: 1073 case AMDGPU::SI_SPILL_V256_RESTORE: 1074 case AMDGPU::SI_SPILL_V512_RESTORE: { 1075 const MachineOperand *VData = TII->getNamedOperand(*MI, 1076 AMDGPU::OpName::vdata); 1077 1078 buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 1079 Index, 1080 VData->getReg(), VData->isKill(), 1081 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1082 TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg(), 1083 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1084 *MI->memoperands_begin(), 1085 RS); 1086 MI->eraseFromParent(); 1087 break; 1088 } 1089 1090 default: { 1091 const DebugLoc &DL = MI->getDebugLoc(); 1092 bool IsMUBUF = TII->isMUBUF(*MI); 1093 1094 if (!IsMUBUF && 1095 MFI->getFrameOffsetReg() != MFI->getScratchWaveOffsetReg()) { 1096 // Convert to an absolute stack address by finding the offset from the 1097 // scratch wave base and scaling by the wave size. 1098 // 1099 // In an entry function/kernel the stack address is already the 1100 // absolute address relative to the scratch wave offset. 1101 1102 unsigned DiffReg 1103 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 1104 1105 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 1106 unsigned ResultReg = IsCopy ? 1107 MI->getOperand(0).getReg() : 1108 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1109 1110 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), DiffReg) 1111 .addReg(MFI->getFrameOffsetReg()) 1112 .addReg(MFI->getScratchWaveOffsetReg()); 1113 1114 int64_t Offset = FrameInfo.getObjectOffset(Index); 1115 if (Offset == 0) { 1116 // XXX - This never happens because of emergency scavenging slot at 0? 1117 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 1118 .addImm(Log2_32(ST.getWavefrontSize())) 1119 .addReg(DiffReg); 1120 } else { 1121 unsigned ScaledReg 1122 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1123 1124 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ScaledReg) 1125 .addImm(Log2_32(ST.getWavefrontSize())) 1126 .addReg(DiffReg, RegState::Kill); 1127 1128 // TODO: Fold if use instruction is another add of a constant. 1129 if (AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 1130 TII->getAddNoCarry(*MBB, MI, DL, ResultReg) 1131 .addImm(Offset) 1132 .addReg(ScaledReg, RegState::Kill) 1133 .addImm(0); // clamp bit 1134 } else { 1135 unsigned ConstOffsetReg 1136 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 1137 1138 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 1139 .addImm(Offset); 1140 TII->getAddNoCarry(*MBB, MI, DL, ResultReg) 1141 .addReg(ConstOffsetReg, RegState::Kill) 1142 .addReg(ScaledReg, RegState::Kill) 1143 .addImm(0); // clamp bit 1144 } 1145 } 1146 1147 // Don't introduce an extra copy if we're just materializing in a mov. 1148 if (IsCopy) 1149 MI->eraseFromParent(); 1150 else 1151 FIOp.ChangeToRegister(ResultReg, false, false, true); 1152 return; 1153 } 1154 1155 if (IsMUBUF) { 1156 // Disable offen so we don't need a 0 vgpr base. 1157 assert(static_cast<int>(FIOperandNum) == 1158 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1159 AMDGPU::OpName::vaddr)); 1160 1161 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() 1162 == MFI->getFrameOffsetReg()); 1163 1164 int64_t Offset = FrameInfo.getObjectOffset(Index); 1165 int64_t OldImm 1166 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 1167 int64_t NewOffset = OldImm + Offset; 1168 1169 if (isUInt<12>(NewOffset) && 1170 buildMUBUFOffsetLoadStore(TII, FrameInfo, MI, Index, NewOffset)) { 1171 MI->eraseFromParent(); 1172 return; 1173 } 1174 } 1175 1176 // If the offset is simply too big, don't convert to a scratch wave offset 1177 // relative index. 1178 1179 int64_t Offset = FrameInfo.getObjectOffset(Index); 1180 FIOp.ChangeToImmediate(Offset); 1181 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 1182 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1183 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 1184 .addImm(Offset); 1185 FIOp.ChangeToRegister(TmpReg, false, false, true); 1186 } 1187 } 1188 } 1189 } 1190 1191 StringRef SIRegisterInfo::getRegAsmName(unsigned Reg) const { 1192 #define AMDGPU_REG_ASM_NAMES 1193 #include "AMDGPURegAsmNames.inc.cpp" 1194 1195 #define REG_RANGE(BeginReg, EndReg, RegTable) \ 1196 if (Reg >= BeginReg && Reg <= EndReg) { \ 1197 unsigned Index = Reg - BeginReg; \ 1198 assert(Index < array_lengthof(RegTable)); \ 1199 return RegTable[Index]; \ 1200 } 1201 1202 REG_RANGE(AMDGPU::VGPR0, AMDGPU::VGPR255, VGPR32RegNames); 1203 REG_RANGE(AMDGPU::SGPR0, AMDGPU::SGPR103, SGPR32RegNames); 1204 REG_RANGE(AMDGPU::VGPR0_VGPR1, AMDGPU::VGPR254_VGPR255, VGPR64RegNames); 1205 REG_RANGE(AMDGPU::SGPR0_SGPR1, AMDGPU::SGPR102_SGPR103, SGPR64RegNames); 1206 REG_RANGE(AMDGPU::VGPR0_VGPR1_VGPR2, AMDGPU::VGPR253_VGPR254_VGPR255, 1207 VGPR96RegNames); 1208 1209 REG_RANGE(AMDGPU::VGPR0_VGPR1_VGPR2_VGPR3, 1210 AMDGPU::VGPR252_VGPR253_VGPR254_VGPR255, 1211 VGPR128RegNames); 1212 REG_RANGE(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, 1213 AMDGPU::SGPR100_SGPR101_SGPR102_SGPR103, 1214 SGPR128RegNames); 1215 1216 REG_RANGE(AMDGPU::VGPR0_VGPR1_VGPR2_VGPR3_VGPR4_VGPR5_VGPR6_VGPR7, 1217 AMDGPU::VGPR248_VGPR249_VGPR250_VGPR251_VGPR252_VGPR253_VGPR254_VGPR255, 1218 VGPR256RegNames); 1219 1220 REG_RANGE( 1221 AMDGPU::VGPR0_VGPR1_VGPR2_VGPR3_VGPR4_VGPR5_VGPR6_VGPR7_VGPR8_VGPR9_VGPR10_VGPR11_VGPR12_VGPR13_VGPR14_VGPR15, 1222 AMDGPU::VGPR240_VGPR241_VGPR242_VGPR243_VGPR244_VGPR245_VGPR246_VGPR247_VGPR248_VGPR249_VGPR250_VGPR251_VGPR252_VGPR253_VGPR254_VGPR255, 1223 VGPR512RegNames); 1224 1225 REG_RANGE(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7, 1226 AMDGPU::SGPR96_SGPR97_SGPR98_SGPR99_SGPR100_SGPR101_SGPR102_SGPR103, 1227 SGPR256RegNames); 1228 1229 REG_RANGE( 1230 AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7_SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15, 1231 AMDGPU::SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95_SGPR96_SGPR97_SGPR98_SGPR99_SGPR100_SGPR101_SGPR102_SGPR103, 1232 SGPR512RegNames 1233 ); 1234 1235 #undef REG_RANGE 1236 1237 // FIXME: Rename flat_scr so we don't need to special case this. 1238 switch (Reg) { 1239 case AMDGPU::FLAT_SCR: 1240 return "flat_scratch"; 1241 case AMDGPU::FLAT_SCR_LO: 1242 return "flat_scratch_lo"; 1243 case AMDGPU::FLAT_SCR_HI: 1244 return "flat_scratch_hi"; 1245 default: 1246 // For the special named registers the default is fine. 1247 return TargetRegisterInfo::getRegAsmName(Reg); 1248 } 1249 } 1250 1251 // FIXME: This is very slow. It might be worth creating a map from physreg to 1252 // register class. 1253 const TargetRegisterClass *SIRegisterInfo::getPhysRegClass(unsigned Reg) const { 1254 assert(!TargetRegisterInfo::isVirtualRegister(Reg)); 1255 1256 static const TargetRegisterClass *const BaseClasses[] = { 1257 &AMDGPU::VGPR_32RegClass, 1258 &AMDGPU::SReg_32RegClass, 1259 &AMDGPU::VReg_64RegClass, 1260 &AMDGPU::SReg_64RegClass, 1261 &AMDGPU::VReg_96RegClass, 1262 &AMDGPU::SReg_96RegClass, 1263 &AMDGPU::VReg_128RegClass, 1264 &AMDGPU::SReg_128RegClass, 1265 &AMDGPU::VReg_160RegClass, 1266 &AMDGPU::SReg_160RegClass, 1267 &AMDGPU::VReg_256RegClass, 1268 &AMDGPU::SReg_256RegClass, 1269 &AMDGPU::VReg_512RegClass, 1270 &AMDGPU::SReg_512RegClass, 1271 &AMDGPU::SCC_CLASSRegClass, 1272 &AMDGPU::Pseudo_SReg_32RegClass, 1273 &AMDGPU::Pseudo_SReg_128RegClass, 1274 }; 1275 1276 for (const TargetRegisterClass *BaseClass : BaseClasses) { 1277 if (BaseClass->contains(Reg)) { 1278 return BaseClass; 1279 } 1280 } 1281 return nullptr; 1282 } 1283 1284 // TODO: It might be helpful to have some target specific flags in 1285 // TargetRegisterClass to mark which classes are VGPRs to make this trivial. 1286 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 1287 unsigned Size = getRegSizeInBits(*RC); 1288 if (Size < 32) 1289 return false; 1290 switch (Size) { 1291 case 32: 1292 return getCommonSubClass(&AMDGPU::VGPR_32RegClass, RC) != nullptr; 1293 case 64: 1294 return getCommonSubClass(&AMDGPU::VReg_64RegClass, RC) != nullptr; 1295 case 96: 1296 return getCommonSubClass(&AMDGPU::VReg_96RegClass, RC) != nullptr; 1297 case 128: 1298 return getCommonSubClass(&AMDGPU::VReg_128RegClass, RC) != nullptr; 1299 case 160: 1300 return getCommonSubClass(&AMDGPU::VReg_160RegClass, RC) != nullptr; 1301 case 256: 1302 return getCommonSubClass(&AMDGPU::VReg_256RegClass, RC) != nullptr; 1303 case 512: 1304 return getCommonSubClass(&AMDGPU::VReg_512RegClass, RC) != nullptr; 1305 default: 1306 llvm_unreachable("Invalid register class size"); 1307 } 1308 } 1309 1310 const TargetRegisterClass *SIRegisterInfo::getEquivalentVGPRClass( 1311 const TargetRegisterClass *SRC) const { 1312 switch (getRegSizeInBits(*SRC)) { 1313 case 32: 1314 return &AMDGPU::VGPR_32RegClass; 1315 case 64: 1316 return &AMDGPU::VReg_64RegClass; 1317 case 96: 1318 return &AMDGPU::VReg_96RegClass; 1319 case 128: 1320 return &AMDGPU::VReg_128RegClass; 1321 case 160: 1322 return &AMDGPU::VReg_160RegClass; 1323 case 256: 1324 return &AMDGPU::VReg_256RegClass; 1325 case 512: 1326 return &AMDGPU::VReg_512RegClass; 1327 default: 1328 llvm_unreachable("Invalid register class size"); 1329 } 1330 } 1331 1332 const TargetRegisterClass *SIRegisterInfo::getEquivalentSGPRClass( 1333 const TargetRegisterClass *VRC) const { 1334 switch (getRegSizeInBits(*VRC)) { 1335 case 32: 1336 return &AMDGPU::SGPR_32RegClass; 1337 case 64: 1338 return &AMDGPU::SReg_64RegClass; 1339 case 96: 1340 return &AMDGPU::SReg_96RegClass; 1341 case 128: 1342 return &AMDGPU::SReg_128RegClass; 1343 case 160: 1344 return &AMDGPU::SReg_160RegClass; 1345 case 256: 1346 return &AMDGPU::SReg_256RegClass; 1347 case 512: 1348 return &AMDGPU::SReg_512RegClass; 1349 default: 1350 llvm_unreachable("Invalid register class size"); 1351 } 1352 } 1353 1354 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 1355 const TargetRegisterClass *RC, unsigned SubIdx) const { 1356 if (SubIdx == AMDGPU::NoSubRegister) 1357 return RC; 1358 1359 // We can assume that each lane corresponds to one 32-bit register. 1360 unsigned Count = getSubRegIndexLaneMask(SubIdx).getNumLanes(); 1361 if (isSGPRClass(RC)) { 1362 switch (Count) { 1363 case 1: 1364 return &AMDGPU::SGPR_32RegClass; 1365 case 2: 1366 return &AMDGPU::SReg_64RegClass; 1367 case 3: 1368 return &AMDGPU::SReg_96RegClass; 1369 case 4: 1370 return &AMDGPU::SReg_128RegClass; 1371 case 5: 1372 return &AMDGPU::SReg_160RegClass; 1373 case 8: 1374 return &AMDGPU::SReg_256RegClass; 1375 case 16: /* fall-through */ 1376 default: 1377 llvm_unreachable("Invalid sub-register class size"); 1378 } 1379 } else { 1380 switch (Count) { 1381 case 1: 1382 return &AMDGPU::VGPR_32RegClass; 1383 case 2: 1384 return &AMDGPU::VReg_64RegClass; 1385 case 3: 1386 return &AMDGPU::VReg_96RegClass; 1387 case 4: 1388 return &AMDGPU::VReg_128RegClass; 1389 case 5: 1390 return &AMDGPU::VReg_160RegClass; 1391 case 8: 1392 return &AMDGPU::VReg_256RegClass; 1393 case 16: /* fall-through */ 1394 default: 1395 llvm_unreachable("Invalid sub-register class size"); 1396 } 1397 } 1398 } 1399 1400 bool SIRegisterInfo::shouldRewriteCopySrc( 1401 const TargetRegisterClass *DefRC, 1402 unsigned DefSubReg, 1403 const TargetRegisterClass *SrcRC, 1404 unsigned SrcSubReg) const { 1405 // We want to prefer the smallest register class possible, so we don't want to 1406 // stop and rewrite on anything that looks like a subregister 1407 // extract. Operations mostly don't care about the super register class, so we 1408 // only want to stop on the most basic of copies between the same register 1409 // class. 1410 // 1411 // e.g. if we have something like 1412 // %0 = ... 1413 // %1 = ... 1414 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 1415 // %3 = COPY %2, sub0 1416 // 1417 // We want to look through the COPY to find: 1418 // => %3 = COPY %0 1419 1420 // Plain copy. 1421 return getCommonSubClass(DefRC, SrcRC) != nullptr; 1422 } 1423 1424 /// Returns a register that is not used at any point in the function. 1425 /// If all registers are used, then this function will return 1426 // AMDGPU::NoRegister. 1427 unsigned 1428 SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 1429 const TargetRegisterClass *RC, 1430 const MachineFunction &MF) const { 1431 1432 for (unsigned Reg : *RC) 1433 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1434 return Reg; 1435 return AMDGPU::NoRegister; 1436 } 1437 1438 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 1439 unsigned EltSize) const { 1440 if (EltSize == 4) { 1441 static const int16_t Sub0_15[] = { 1442 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1443 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1444 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1445 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1446 }; 1447 1448 static const int16_t Sub0_7[] = { 1449 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1450 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1451 }; 1452 1453 static const int16_t Sub0_4[] = { 1454 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, AMDGPU::sub4, 1455 }; 1456 1457 static const int16_t Sub0_3[] = { 1458 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1459 }; 1460 1461 static const int16_t Sub0_2[] = { 1462 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, 1463 }; 1464 1465 static const int16_t Sub0_1[] = { 1466 AMDGPU::sub0, AMDGPU::sub1, 1467 }; 1468 1469 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1470 case 32: 1471 return {}; 1472 case 64: 1473 return makeArrayRef(Sub0_1); 1474 case 96: 1475 return makeArrayRef(Sub0_2); 1476 case 128: 1477 return makeArrayRef(Sub0_3); 1478 case 160: 1479 return makeArrayRef(Sub0_4); 1480 case 256: 1481 return makeArrayRef(Sub0_7); 1482 case 512: 1483 return makeArrayRef(Sub0_15); 1484 default: 1485 llvm_unreachable("unhandled register size"); 1486 } 1487 } 1488 1489 if (EltSize == 8) { 1490 static const int16_t Sub0_15_64[] = { 1491 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1492 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1493 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1494 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15 1495 }; 1496 1497 static const int16_t Sub0_7_64[] = { 1498 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1499 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7 1500 }; 1501 1502 1503 static const int16_t Sub0_3_64[] = { 1504 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3 1505 }; 1506 1507 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1508 case 64: 1509 return {}; 1510 case 128: 1511 return makeArrayRef(Sub0_3_64); 1512 case 256: 1513 return makeArrayRef(Sub0_7_64); 1514 case 512: 1515 return makeArrayRef(Sub0_15_64); 1516 default: 1517 llvm_unreachable("unhandled register size"); 1518 } 1519 } 1520 1521 assert(EltSize == 16 && "unhandled register spill split size"); 1522 1523 static const int16_t Sub0_15_128[] = { 1524 AMDGPU::sub0_sub1_sub2_sub3, 1525 AMDGPU::sub4_sub5_sub6_sub7, 1526 AMDGPU::sub8_sub9_sub10_sub11, 1527 AMDGPU::sub12_sub13_sub14_sub15 1528 }; 1529 1530 static const int16_t Sub0_7_128[] = { 1531 AMDGPU::sub0_sub1_sub2_sub3, 1532 AMDGPU::sub4_sub5_sub6_sub7 1533 }; 1534 1535 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1536 case 128: 1537 return {}; 1538 case 256: 1539 return makeArrayRef(Sub0_7_128); 1540 case 512: 1541 return makeArrayRef(Sub0_15_128); 1542 default: 1543 llvm_unreachable("unhandled register size"); 1544 } 1545 } 1546 1547 const TargetRegisterClass* 1548 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 1549 unsigned Reg) const { 1550 if (TargetRegisterInfo::isVirtualRegister(Reg)) 1551 return MRI.getRegClass(Reg); 1552 1553 return getPhysRegClass(Reg); 1554 } 1555 1556 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 1557 unsigned Reg) const { 1558 const TargetRegisterClass * RC = getRegClassForReg(MRI, Reg); 1559 assert(RC && "Register class for the reg not found"); 1560 return hasVGPRs(RC); 1561 } 1562 1563 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 1564 const TargetRegisterClass *SrcRC, 1565 unsigned SubReg, 1566 const TargetRegisterClass *DstRC, 1567 unsigned DstSubReg, 1568 const TargetRegisterClass *NewRC, 1569 LiveIntervals &LIS) const { 1570 unsigned SrcSize = getRegSizeInBits(*SrcRC); 1571 unsigned DstSize = getRegSizeInBits(*DstRC); 1572 unsigned NewSize = getRegSizeInBits(*NewRC); 1573 1574 // Do not increase size of registers beyond dword, we would need to allocate 1575 // adjacent registers and constraint regalloc more than needed. 1576 1577 // Always allow dword coalescing. 1578 if (SrcSize <= 32 || DstSize <= 32) 1579 return true; 1580 1581 return NewSize <= DstSize || NewSize <= SrcSize; 1582 } 1583 1584 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1585 MachineFunction &MF) const { 1586 1587 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 1588 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1589 1590 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 1591 MF.getFunction()); 1592 switch (RC->getID()) { 1593 default: 1594 return AMDGPURegisterInfo::getRegPressureLimit(RC, MF); 1595 case AMDGPU::VGPR_32RegClassID: 1596 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 1597 case AMDGPU::SGPR_32RegClassID: 1598 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 1599 } 1600 } 1601 1602 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 1603 unsigned Idx) const { 1604 if (Idx == getVGPRPressureSet()) 1605 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 1606 const_cast<MachineFunction &>(MF)); 1607 1608 if (Idx == getSGPRPressureSet()) 1609 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 1610 const_cast<MachineFunction &>(MF)); 1611 1612 return AMDGPURegisterInfo::getRegPressureSetLimit(MF, Idx); 1613 } 1614 1615 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 1616 static const int Empty[] = { -1 }; 1617 1618 if (hasRegUnit(AMDGPU::M0, RegUnit)) 1619 return Empty; 1620 return AMDGPURegisterInfo::getRegUnitPressureSets(RegUnit); 1621 } 1622 1623 unsigned SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 1624 // Not a callee saved register. 1625 return AMDGPU::SGPR30_SGPR31; 1626 } 1627 1628 const TargetRegisterClass * 1629 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 1630 const MachineRegisterInfo &MRI) const { 1631 unsigned Size = getRegSizeInBits(MO.getReg(), MRI); 1632 const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg()); 1633 if (!RB) 1634 return nullptr; 1635 1636 Size = PowerOf2Ceil(Size); 1637 switch (Size) { 1638 case 32: 1639 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VGPR_32RegClass : 1640 &AMDGPU::SReg_32_XM0RegClass; 1641 case 64: 1642 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_64RegClass : 1643 &AMDGPU::SReg_64_XEXECRegClass; 1644 case 96: 1645 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_96RegClass : 1646 &AMDGPU::SReg_96RegClass; 1647 case 128: 1648 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_128RegClass : 1649 &AMDGPU::SReg_128RegClass; 1650 case 160: 1651 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_160RegClass : 1652 &AMDGPU::SReg_160RegClass; 1653 case 256: 1654 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_256RegClass : 1655 &AMDGPU::SReg_256RegClass; 1656 case 512: 1657 return RB->getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_512RegClass : 1658 &AMDGPU::SReg_512RegClass; 1659 default: 1660 llvm_unreachable("not implemented"); 1661 } 1662 } 1663 1664 // Find reaching register definition 1665 MachineInstr *SIRegisterInfo::findReachingDef(unsigned Reg, unsigned SubReg, 1666 MachineInstr &Use, 1667 MachineRegisterInfo &MRI, 1668 LiveIntervals *LIS) const { 1669 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 1670 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 1671 SlotIndex DefIdx; 1672 1673 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 1674 if (!LIS->hasInterval(Reg)) 1675 return nullptr; 1676 LiveInterval &LI = LIS->getInterval(Reg); 1677 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 1678 : MRI.getMaxLaneMaskForVReg(Reg); 1679 VNInfo *V = nullptr; 1680 if (LI.hasSubRanges()) { 1681 for (auto &S : LI.subranges()) { 1682 if ((S.LaneMask & SubLanes) == SubLanes) { 1683 V = S.getVNInfoAt(UseIdx); 1684 break; 1685 } 1686 } 1687 } else { 1688 V = LI.getVNInfoAt(UseIdx); 1689 } 1690 if (!V) 1691 return nullptr; 1692 DefIdx = V->def; 1693 } else { 1694 // Find last def. 1695 for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) { 1696 LiveRange &LR = LIS->getRegUnit(*Units); 1697 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 1698 if (!DefIdx.isValid() || 1699 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 1700 LIS->getInstructionFromIndex(V->def))) 1701 DefIdx = V->def; 1702 } else { 1703 return nullptr; 1704 } 1705 } 1706 } 1707 1708 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 1709 1710 if (!Def || !MDT.dominates(Def, &Use)) 1711 return nullptr; 1712 1713 assert(Def->modifiesRegister(Reg, this)); 1714 1715 return Def; 1716 } 1717