1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief SI implementation of the TargetRegisterInfo class.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "SIRegisterInfo.h"
16 #include "SIInstrInfo.h"
17 #include "SIMachineFunctionInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineInstrBuilder.h"
21 #include "llvm/CodeGen/RegisterScavenging.h"
22 #include "llvm/IR/Function.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/Support/MathExtras.h"
25 
26 using namespace llvm;
27 
28 static bool hasPressureSet(const int *PSets, unsigned PSetID) {
29   for (unsigned i = 0; PSets[i] != -1; ++i) {
30     if (PSets[i] == (int)PSetID)
31       return true;
32   }
33   return false;
34 }
35 
36 void SIRegisterInfo::classifyPressureSet(unsigned PSetID, unsigned Reg,
37                                          BitVector &PressureSets) const {
38   for (MCRegUnitIterator U(Reg, this); U.isValid(); ++U) {
39     const int *PSets = getRegUnitPressureSets(*U);
40     if (hasPressureSet(PSets, PSetID)) {
41       PressureSets.set(PSetID);
42       break;
43     }
44   }
45 }
46 
47 static cl::opt<bool> EnableSpillSGPRToSMEM(
48   "amdgpu-spill-sgpr-to-smem",
49   cl::desc("Use scalar stores to spill SGPRs if supported by subtarget"),
50   cl::init(false));
51 
52 static cl::opt<bool> EnableSpillSGPRToVGPR(
53   "amdgpu-spill-sgpr-to-vgpr",
54   cl::desc("Enable spilling VGPRs to SGPRs"),
55   cl::ReallyHidden,
56   cl::init(true));
57 
58 SIRegisterInfo::SIRegisterInfo(const SISubtarget &ST) :
59   AMDGPURegisterInfo(),
60   SGPRPressureSets(getNumRegPressureSets()),
61   VGPRPressureSets(getNumRegPressureSets()),
62   SpillSGPRToVGPR(false),
63   SpillSGPRToSMEM(false) {
64   if (EnableSpillSGPRToSMEM && ST.hasScalarStores())
65     SpillSGPRToSMEM = true;
66   else if (EnableSpillSGPRToVGPR)
67     SpillSGPRToVGPR = true;
68 
69   unsigned NumRegPressureSets = getNumRegPressureSets();
70 
71   SGPRSetID = NumRegPressureSets;
72   VGPRSetID = NumRegPressureSets;
73 
74   for (unsigned i = 0; i < NumRegPressureSets; ++i) {
75     classifyPressureSet(i, AMDGPU::SGPR0, SGPRPressureSets);
76     classifyPressureSet(i, AMDGPU::VGPR0, VGPRPressureSets);
77   }
78 
79   // Determine the number of reg units for each pressure set.
80   std::vector<unsigned> PressureSetRegUnits(NumRegPressureSets, 0);
81   for (unsigned i = 0, e = getNumRegUnits(); i != e; ++i) {
82     const int *PSets = getRegUnitPressureSets(i);
83     for (unsigned j = 0; PSets[j] != -1; ++j) {
84       ++PressureSetRegUnits[PSets[j]];
85     }
86   }
87 
88   unsigned VGPRMax = 0, SGPRMax = 0;
89   for (unsigned i = 0; i < NumRegPressureSets; ++i) {
90     if (isVGPRPressureSet(i) && PressureSetRegUnits[i] > VGPRMax) {
91       VGPRSetID = i;
92       VGPRMax = PressureSetRegUnits[i];
93       continue;
94     }
95     if (isSGPRPressureSet(i) && PressureSetRegUnits[i] > SGPRMax) {
96       SGPRSetID = i;
97       SGPRMax = PressureSetRegUnits[i];
98     }
99   }
100 
101   assert(SGPRSetID < NumRegPressureSets &&
102          VGPRSetID < NumRegPressureSets);
103 }
104 
105 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, unsigned Reg) const {
106   MCRegAliasIterator R(Reg, this, true);
107 
108   for (; R.isValid(); ++R)
109     Reserved.set(*R);
110 }
111 
112 unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg(
113   const MachineFunction &MF) const {
114 
115   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
116   unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
117   unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
118   return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass);
119 }
120 
121 unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg(
122   const MachineFunction &MF) const {
123 
124   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
125   unsigned RegCount = ST.getMaxNumSGPRs(MF);
126   unsigned Reg;
127 
128   // Try to place it in a hole after PrivateSegmentbufferReg.
129   if (RegCount & 3) {
130     // We cannot put the segment buffer in (Idx - 4) ... (Idx - 1) due to
131     // alignment constraints, so we have a hole where can put the wave offset.
132     Reg = RegCount - 1;
133   } else {
134     // We can put the segment buffer in (Idx - 4) ... (Idx - 1) and put the
135     // wave offset before it.
136     Reg = RegCount - 5;
137   }
138   return AMDGPU::SGPR_32RegClass.getRegister(Reg);
139 }
140 
141 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
142   BitVector Reserved(getNumRegs());
143   Reserved.set(AMDGPU::INDIRECT_BASE_ADDR);
144 
145   // EXEC_LO and EXEC_HI could be allocated and used as regular register, but
146   // this seems likely to result in bugs, so I'm marking them as reserved.
147   reserveRegisterTuples(Reserved, AMDGPU::EXEC);
148   reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR);
149 
150   // Reserve the memory aperture registers.
151   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE);
152   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT);
153   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
154   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
155 
156   // Reserve Trap Handler registers - support is not implemented in Codegen.
157   reserveRegisterTuples(Reserved, AMDGPU::TBA);
158   reserveRegisterTuples(Reserved, AMDGPU::TMA);
159   reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1);
160   reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3);
161   reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5);
162   reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7);
163   reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9);
164   reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11);
165 
166   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
167 
168   unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
169   unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
170   for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) {
171     unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
172     reserveRegisterTuples(Reserved, Reg);
173   }
174 
175   unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF);
176   unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
177   for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) {
178     unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i);
179     reserveRegisterTuples(Reserved, Reg);
180   }
181 
182   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
183 
184   unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
185   if (ScratchWaveOffsetReg != AMDGPU::NoRegister) {
186     // Reserve 1 SGPR for scratch wave offset in case we need to spill.
187     reserveRegisterTuples(Reserved, ScratchWaveOffsetReg);
188   }
189 
190   unsigned ScratchRSrcReg = MFI->getScratchRSrcReg();
191   if (ScratchRSrcReg != AMDGPU::NoRegister) {
192     // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need
193     // to spill.
194     // TODO: May need to reserve a VGPR if doing LDS spilling.
195     reserveRegisterTuples(Reserved, ScratchRSrcReg);
196     assert(!isSubRegister(ScratchRSrcReg, ScratchWaveOffsetReg));
197   }
198 
199   return Reserved;
200 }
201 
202 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const {
203   return Fn.getFrameInfo().hasStackObjects();
204 }
205 
206 bool
207 SIRegisterInfo::requiresFrameIndexScavenging(const MachineFunction &MF) const {
208   return MF.getFrameInfo().hasStackObjects();
209 }
210 
211 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging(
212   const MachineFunction &MF) const {
213   // m0 is needed for the scalar store offset. m0 is unallocatable, so we can't
214   // create a virtual register for it during frame index elimination, so the
215   // scavenger is directly needed.
216   return MF.getFrameInfo().hasStackObjects() &&
217          MF.getSubtarget<SISubtarget>().hasScalarStores() &&
218          MF.getInfo<SIMachineFunctionInfo>()->hasSpilledSGPRs();
219 }
220 
221 bool SIRegisterInfo::requiresVirtualBaseRegisters(
222   const MachineFunction &) const {
223   // There are no special dedicated stack or frame pointers.
224   return true;
225 }
226 
227 bool SIRegisterInfo::trackLivenessAfterRegAlloc(const MachineFunction &MF) const {
228   // This helps catch bugs as verifier errors.
229   return true;
230 }
231 
232 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const {
233   assert(SIInstrInfo::isMUBUF(*MI));
234 
235   int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
236                                           AMDGPU::OpName::offset);
237   return MI->getOperand(OffIdx).getImm();
238 }
239 
240 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI,
241                                                  int Idx) const {
242   if (!SIInstrInfo::isMUBUF(*MI))
243     return 0;
244 
245   assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(),
246                                            AMDGPU::OpName::vaddr) &&
247          "Should never see frame index on non-address operand");
248 
249   return getMUBUFInstrOffset(MI);
250 }
251 
252 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
253   if (!MI->mayLoadOrStore())
254     return false;
255 
256   int64_t FullOffset = Offset + getMUBUFInstrOffset(MI);
257 
258   return !isUInt<12>(FullOffset);
259 }
260 
261 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
262                                                   unsigned BaseReg,
263                                                   int FrameIdx,
264                                                   int64_t Offset) const {
265   MachineBasicBlock::iterator Ins = MBB->begin();
266   DebugLoc DL; // Defaults to "unknown"
267 
268   if (Ins != MBB->end())
269     DL = Ins->getDebugLoc();
270 
271   MachineFunction *MF = MBB->getParent();
272   const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>();
273   const SIInstrInfo *TII = Subtarget.getInstrInfo();
274 
275   if (Offset == 0) {
276     BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg)
277       .addFrameIndex(FrameIdx);
278     return;
279   }
280 
281   MachineRegisterInfo &MRI = MF->getRegInfo();
282   unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
283   unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
284 
285   unsigned FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
286 
287   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
288     .addImm(Offset);
289   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg)
290     .addFrameIndex(FrameIdx);
291 
292   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_ADD_I32_e64), BaseReg)
293     .addReg(UnusedCarry, RegState::Define | RegState::Dead)
294     .addReg(OffsetReg, RegState::Kill)
295     .addReg(FIReg);
296 }
297 
298 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg,
299                                        int64_t Offset) const {
300 
301   MachineBasicBlock *MBB = MI.getParent();
302   MachineFunction *MF = MBB->getParent();
303   const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>();
304   const SIInstrInfo *TII = Subtarget.getInstrInfo();
305 
306 #ifndef NDEBUG
307   // FIXME: Is it possible to be storing a frame index to itself?
308   bool SeenFI = false;
309   for (const MachineOperand &MO: MI.operands()) {
310     if (MO.isFI()) {
311       if (SeenFI)
312         llvm_unreachable("should not see multiple frame indices");
313 
314       SeenFI = true;
315     }
316   }
317 #endif
318 
319   MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
320   assert(FIOp && FIOp->isFI() && "frame index must be address operand");
321 
322   assert(TII->isMUBUF(MI));
323 
324   MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset);
325   int64_t NewOffset = OffsetOp->getImm() + Offset;
326   assert(isUInt<12>(NewOffset) && "offset should be legal");
327 
328   FIOp->ChangeToRegister(BaseReg, false);
329   OffsetOp->setImm(NewOffset);
330 }
331 
332 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
333                                         unsigned BaseReg,
334                                         int64_t Offset) const {
335   if (!SIInstrInfo::isMUBUF(*MI))
336     return false;
337 
338   int64_t NewOffset = Offset + getMUBUFInstrOffset(MI);
339 
340   return isUInt<12>(NewOffset);
341 }
342 
343 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass(
344   const MachineFunction &MF, unsigned Kind) const {
345   // This is inaccurate. It depends on the instruction and address space. The
346   // only place where we should hit this is for dealing with frame indexes /
347   // private accesses, so this is correct in that case.
348   return &AMDGPU::VGPR_32RegClass;
349 }
350 
351 static unsigned getNumSubRegsForSpillOp(unsigned Op) {
352 
353   switch (Op) {
354   case AMDGPU::SI_SPILL_S512_SAVE:
355   case AMDGPU::SI_SPILL_S512_RESTORE:
356   case AMDGPU::SI_SPILL_V512_SAVE:
357   case AMDGPU::SI_SPILL_V512_RESTORE:
358     return 16;
359   case AMDGPU::SI_SPILL_S256_SAVE:
360   case AMDGPU::SI_SPILL_S256_RESTORE:
361   case AMDGPU::SI_SPILL_V256_SAVE:
362   case AMDGPU::SI_SPILL_V256_RESTORE:
363     return 8;
364   case AMDGPU::SI_SPILL_S128_SAVE:
365   case AMDGPU::SI_SPILL_S128_RESTORE:
366   case AMDGPU::SI_SPILL_V128_SAVE:
367   case AMDGPU::SI_SPILL_V128_RESTORE:
368     return 4;
369   case AMDGPU::SI_SPILL_V96_SAVE:
370   case AMDGPU::SI_SPILL_V96_RESTORE:
371     return 3;
372   case AMDGPU::SI_SPILL_S64_SAVE:
373   case AMDGPU::SI_SPILL_S64_RESTORE:
374   case AMDGPU::SI_SPILL_V64_SAVE:
375   case AMDGPU::SI_SPILL_V64_RESTORE:
376     return 2;
377   case AMDGPU::SI_SPILL_S32_SAVE:
378   case AMDGPU::SI_SPILL_S32_RESTORE:
379   case AMDGPU::SI_SPILL_V32_SAVE:
380   case AMDGPU::SI_SPILL_V32_RESTORE:
381     return 1;
382   default: llvm_unreachable("Invalid spill opcode");
383   }
384 }
385 
386 static int getOffsetMUBUFStore(unsigned Opc) {
387   switch (Opc) {
388   case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
389     return AMDGPU::BUFFER_STORE_DWORD_OFFSET;
390   case AMDGPU::BUFFER_STORE_BYTE_OFFEN:
391     return AMDGPU::BUFFER_STORE_BYTE_OFFSET;
392   case AMDGPU::BUFFER_STORE_SHORT_OFFEN:
393     return AMDGPU::BUFFER_STORE_SHORT_OFFSET;
394   case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN:
395     return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET;
396   case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN:
397     return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET;
398   default:
399     return -1;
400   }
401 }
402 
403 static int getOffsetMUBUFLoad(unsigned Opc) {
404   switch (Opc) {
405   case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
406     return AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
407   case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN:
408     return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET;
409   case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN:
410     return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET;
411   case AMDGPU::BUFFER_LOAD_USHORT_OFFEN:
412     return AMDGPU::BUFFER_LOAD_USHORT_OFFSET;
413   case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN:
414     return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET;
415   case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN:
416     return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET;
417   case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN:
418     return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET;
419   default:
420     return -1;
421   }
422 }
423 
424 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not
425 // need to handle the case where an SGPR may need to be spilled while spilling.
426 static bool buildMUBUFOffsetLoadStore(const SIInstrInfo *TII,
427                                       MachineFrameInfo &MFI,
428                                       MachineBasicBlock::iterator MI,
429                                       int Index,
430                                       int64_t Offset) {
431   MachineBasicBlock *MBB = MI->getParent();
432   const DebugLoc &DL = MI->getDebugLoc();
433   bool IsStore = MI->mayStore();
434 
435   unsigned Opc = MI->getOpcode();
436   int LoadStoreOp = IsStore ?
437     getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc);
438   if (LoadStoreOp == -1)
439     return false;
440 
441   unsigned Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata)->getReg();
442 
443   BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp))
444       .addReg(Reg, getDefRegState(!IsStore))
445       .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc))
446       .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset))
447       .addImm(Offset)
448       .addImm(0) // glc
449       .addImm(0) // slc
450       .addImm(0) // tfe
451       .setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
452   return true;
453 }
454 
455 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
456                                          unsigned LoadStoreOp,
457                                          int Index,
458                                          unsigned ValueReg,
459                                          bool IsKill,
460                                          unsigned ScratchRsrcReg,
461                                          unsigned ScratchOffsetReg,
462                                          int64_t InstOffset,
463                                          MachineMemOperand *MMO,
464                                          RegScavenger *RS) const {
465   MachineBasicBlock *MBB = MI->getParent();
466   MachineFunction *MF = MI->getParent()->getParent();
467   const SISubtarget &ST =  MF->getSubtarget<SISubtarget>();
468   const SIInstrInfo *TII = ST.getInstrInfo();
469   const MachineFrameInfo &MFI = MF->getFrameInfo();
470 
471   const MCInstrDesc &Desc = TII->get(LoadStoreOp);
472   const DebugLoc &DL = MI->getDebugLoc();
473   bool IsStore = Desc.mayStore();
474 
475   bool RanOutOfSGPRs = false;
476   bool Scavenged = false;
477   unsigned SOffset = ScratchOffsetReg;
478 
479   const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg);
480   unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / 32;
481   unsigned Size = NumSubRegs * 4;
482   int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
483   const int64_t OriginalImmOffset = Offset;
484 
485   unsigned Align = MFI.getObjectAlignment(Index);
486   const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo();
487 
488   if (!isUInt<12>(Offset + Size)) {
489     SOffset = AMDGPU::NoRegister;
490 
491     // We don't have access to the register scavenger if this function is called
492     // during  PEI::scavengeFrameVirtualRegs().
493     if (RS)
494       SOffset = RS->FindUnusedReg(&AMDGPU::SGPR_32RegClass);
495 
496     if (SOffset == AMDGPU::NoRegister) {
497       // There are no free SGPRs, and since we are in the process of spilling
498       // VGPRs too.  Since we need a VGPR in order to spill SGPRs (this is true
499       // on SI/CI and on VI it is true until we implement spilling using scalar
500       // stores), we have no way to free up an SGPR.  Our solution here is to
501       // add the offset directly to the ScratchOffset register, and then
502       // subtract the offset after the spill to return ScratchOffset to it's
503       // original value.
504       RanOutOfSGPRs = true;
505       SOffset = ScratchOffsetReg;
506     } else {
507       Scavenged = true;
508     }
509 
510     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset)
511       .addReg(ScratchOffsetReg)
512       .addImm(Offset);
513 
514     Offset = 0;
515   }
516 
517   const unsigned EltSize = 4;
518 
519   for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) {
520     unsigned SubReg = NumSubRegs == 1 ?
521       ValueReg : getSubReg(ValueReg, getSubRegFromChannel(i));
522 
523     unsigned SOffsetRegState = 0;
524     unsigned SrcDstRegState = getDefRegState(!IsStore);
525     if (i + 1 == e) {
526       SOffsetRegState |= getKillRegState(Scavenged);
527       // The last implicit use carries the "Kill" flag.
528       SrcDstRegState |= getKillRegState(IsKill);
529     }
530 
531     MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i);
532     MachineMemOperand *NewMMO
533       = MF->getMachineMemOperand(PInfo, MMO->getFlags(),
534                                  EltSize, MinAlign(Align, EltSize * i));
535 
536     auto MIB = BuildMI(*MBB, MI, DL, Desc)
537       .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill))
538       .addReg(ScratchRsrcReg)
539       .addReg(SOffset, SOffsetRegState)
540       .addImm(Offset)
541       .addImm(0) // glc
542       .addImm(0) // slc
543       .addImm(0) // tfe
544       .addMemOperand(NewMMO);
545 
546     if (NumSubRegs > 1)
547       MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState);
548   }
549 
550   if (RanOutOfSGPRs) {
551     // Subtract the offset we added to the ScratchOffset register.
552     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScratchOffsetReg)
553       .addReg(ScratchOffsetReg)
554       .addImm(OriginalImmOffset);
555   }
556 }
557 
558 static std::pair<unsigned, unsigned> getSpillEltSize(unsigned SuperRegSize,
559                                                      bool Store) {
560   if (SuperRegSize % 16 == 0) {
561     return { 16, Store ? AMDGPU::S_BUFFER_STORE_DWORDX4_SGPR :
562                          AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR };
563   }
564 
565   if (SuperRegSize % 8 == 0) {
566     return { 8, Store ? AMDGPU::S_BUFFER_STORE_DWORDX2_SGPR :
567                         AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR };
568   }
569 
570   return { 4, Store ? AMDGPU::S_BUFFER_STORE_DWORD_SGPR :
571                       AMDGPU::S_BUFFER_LOAD_DWORD_SGPR};
572 }
573 
574 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
575                                int Index,
576                                RegScavenger *RS,
577                                bool OnlyToVGPR) const {
578   MachineBasicBlock *MBB = MI->getParent();
579   MachineFunction *MF = MBB->getParent();
580   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
581 
582   ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
583     = MFI->getSGPRToVGPRSpills(Index);
584   bool SpillToVGPR = !VGPRSpills.empty();
585   if (OnlyToVGPR && !SpillToVGPR)
586     return false;
587 
588   MachineRegisterInfo &MRI = MF->getRegInfo();
589   const SISubtarget &ST =  MF->getSubtarget<SISubtarget>();
590   const SIInstrInfo *TII = ST.getInstrInfo();
591 
592   unsigned SuperReg = MI->getOperand(0).getReg();
593   bool IsKill = MI->getOperand(0).isKill();
594   const DebugLoc &DL = MI->getDebugLoc();
595 
596   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
597 
598   bool SpillToSMEM = spillSGPRToSMEM();
599   if (SpillToSMEM && OnlyToVGPR)
600     return false;
601 
602   assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
603 
604   unsigned OffsetReg = AMDGPU::M0;
605   unsigned M0CopyReg = AMDGPU::NoRegister;
606 
607   if (SpillToSMEM) {
608     if (RS->isRegUsed(AMDGPU::M0)) {
609       M0CopyReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
610       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), M0CopyReg)
611         .addReg(AMDGPU::M0);
612     }
613   }
614 
615   unsigned ScalarStoreOp;
616   unsigned EltSize = 4;
617   const TargetRegisterClass *RC = getPhysRegClass(SuperReg);
618   if (SpillToSMEM && isSGPRClass(RC)) {
619     // XXX - if private_element_size is larger than 4 it might be useful to be
620     // able to spill wider vmem spills.
621     std::tie(EltSize, ScalarStoreOp) = getSpillEltSize(RC->getSize(), true);
622   }
623 
624   ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize);
625   unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
626 
627   // SubReg carries the "Kill" flag when SubReg == SuperReg.
628   unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill);
629   for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
630     unsigned SubReg = NumSubRegs == 1 ?
631       SuperReg : getSubReg(SuperReg, SplitParts[i]);
632 
633     if (SpillToSMEM) {
634       int64_t FrOffset = FrameInfo.getObjectOffset(Index);
635 
636       // The allocated memory size is really the wavefront size * the frame
637       // index size. The widest register class is 64 bytes, so a 4-byte scratch
638       // allocation is enough to spill this in a single stack object.
639       //
640       // FIXME: Frame size/offsets are computed earlier than this, so the extra
641       // space is still unnecessarily allocated.
642 
643       unsigned Align = FrameInfo.getObjectAlignment(Index);
644       MachinePointerInfo PtrInfo
645         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
646       MachineMemOperand *MMO
647         = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
648                                    EltSize, MinAlign(Align, EltSize * i));
649 
650       // SMEM instructions only support a single offset, so increment the wave
651       // offset.
652 
653       int64_t Offset = (ST.getWavefrontSize() * FrOffset) + (EltSize * i);
654       if (Offset != 0) {
655         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), OffsetReg)
656           .addReg(MFI->getScratchWaveOffsetReg())
657           .addImm(Offset);
658       } else {
659         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
660           .addReg(MFI->getScratchWaveOffsetReg());
661       }
662 
663       BuildMI(*MBB, MI, DL, TII->get(ScalarStoreOp))
664         .addReg(SubReg, getKillRegState(IsKill)) // sdata
665         .addReg(MFI->getScratchRSrcReg())        // sbase
666         .addReg(OffsetReg, RegState::Kill)       // soff
667         .addImm(0)                               // glc
668         .addMemOperand(MMO);
669 
670       continue;
671     }
672 
673     if (SpillToVGPR) {
674       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
675 
676       BuildMI(*MBB, MI, DL,
677               TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
678               Spill.VGPR)
679         .addReg(SubReg, getKillRegState(IsKill))
680         .addImm(Spill.Lane);
681 
682       // FIXME: Since this spills to another register instead of an actual
683       // frame index, we should delete the frame index when all references to
684       // it are fixed.
685     } else {
686       // XXX - Can to VGPR spill fail for some subregisters but not others?
687       if (OnlyToVGPR)
688         return false;
689 
690       // Spill SGPR to a frame index.
691       // TODO: Should VI try to spill to VGPR and then spill to SMEM?
692       unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
693       // TODO: Should VI try to spill to VGPR and then spill to SMEM?
694 
695       MachineInstrBuilder Mov
696         = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
697         .addReg(SubReg, SubKillState);
698 
699 
700       // There could be undef components of a spilled super register.
701       // TODO: Can we detect this and skip the spill?
702       if (NumSubRegs > 1) {
703         // The last implicit use of the SuperReg carries the "Kill" flag.
704         unsigned SuperKillState = 0;
705         if (i + 1 == e)
706           SuperKillState |= getKillRegState(IsKill);
707         Mov.addReg(SuperReg, RegState::Implicit | SuperKillState);
708       }
709 
710       unsigned Align = FrameInfo.getObjectAlignment(Index);
711       MachinePointerInfo PtrInfo
712         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
713       MachineMemOperand *MMO
714         = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
715                                    EltSize, MinAlign(Align, EltSize * i));
716       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE))
717         .addReg(TmpReg, RegState::Kill)         // src
718         .addFrameIndex(Index)                   // vaddr
719         .addReg(MFI->getScratchRSrcReg())       // srrsrc
720         .addReg(MFI->getScratchWaveOffsetReg()) // soffset
721         .addImm(i * 4)                          // offset
722         .addMemOperand(MMO);
723     }
724   }
725 
726   if (M0CopyReg != AMDGPU::NoRegister) {
727     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
728       .addReg(M0CopyReg, RegState::Kill);
729   }
730 
731   MI->eraseFromParent();
732   MFI->addToSpilledSGPRs(NumSubRegs);
733   return true;
734 }
735 
736 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
737                                  int Index,
738                                  RegScavenger *RS,
739                                  bool OnlyToVGPR) const {
740   MachineFunction *MF = MI->getParent()->getParent();
741   MachineRegisterInfo &MRI = MF->getRegInfo();
742   MachineBasicBlock *MBB = MI->getParent();
743   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
744 
745   ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
746     = MFI->getSGPRToVGPRSpills(Index);
747   bool SpillToVGPR = !VGPRSpills.empty();
748   if (OnlyToVGPR && !SpillToVGPR)
749     return false;
750 
751   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
752   const SISubtarget &ST =  MF->getSubtarget<SISubtarget>();
753   const SIInstrInfo *TII = ST.getInstrInfo();
754   const DebugLoc &DL = MI->getDebugLoc();
755 
756   unsigned SuperReg = MI->getOperand(0).getReg();
757   bool SpillToSMEM = spillSGPRToSMEM();
758   if (SpillToSMEM && OnlyToVGPR)
759     return false;
760 
761   assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
762 
763   unsigned OffsetReg = AMDGPU::M0;
764   unsigned M0CopyReg = AMDGPU::NoRegister;
765 
766   if (SpillToSMEM) {
767     if (RS->isRegUsed(AMDGPU::M0)) {
768       M0CopyReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
769       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), M0CopyReg)
770         .addReg(AMDGPU::M0);
771     }
772   }
773 
774   unsigned EltSize = 4;
775   unsigned ScalarLoadOp;
776 
777   const TargetRegisterClass *RC = getPhysRegClass(SuperReg);
778   if (SpillToSMEM && isSGPRClass(RC)) {
779     // XXX - if private_element_size is larger than 4 it might be useful to be
780     // able to spill wider vmem spills.
781     std::tie(EltSize, ScalarLoadOp) = getSpillEltSize(RC->getSize(), false);
782   }
783 
784   ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize);
785   unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
786 
787   // SubReg carries the "Kill" flag when SubReg == SuperReg.
788   int64_t FrOffset = FrameInfo.getObjectOffset(Index);
789 
790   for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
791     unsigned SubReg = NumSubRegs == 1 ?
792       SuperReg : getSubReg(SuperReg, SplitParts[i]);
793 
794     if (SpillToSMEM) {
795       // FIXME: Size may be > 4 but extra bytes wasted.
796       unsigned Align = FrameInfo.getObjectAlignment(Index);
797       MachinePointerInfo PtrInfo
798         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
799       MachineMemOperand *MMO
800         = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
801                                    EltSize, MinAlign(Align, EltSize * i));
802 
803       // Add i * 4 offset
804       int64_t Offset = (ST.getWavefrontSize() * FrOffset) + (EltSize * i);
805       if (Offset != 0) {
806         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), OffsetReg)
807           .addReg(MFI->getScratchWaveOffsetReg())
808           .addImm(Offset);
809       } else {
810         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
811           .addReg(MFI->getScratchWaveOffsetReg());
812       }
813 
814       auto MIB =
815         BuildMI(*MBB, MI, DL, TII->get(ScalarLoadOp), SubReg)
816         .addReg(MFI->getScratchRSrcReg()) // sbase
817         .addReg(OffsetReg, RegState::Kill)                // soff
818         .addImm(0)                        // glc
819         .addMemOperand(MMO);
820 
821       if (NumSubRegs > 1)
822         MIB.addReg(SuperReg, RegState::ImplicitDefine);
823 
824       continue;
825     }
826 
827     if (SpillToVGPR) {
828       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
829       auto MIB =
830         BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
831                 SubReg)
832         .addReg(Spill.VGPR)
833         .addImm(Spill.Lane);
834 
835       if (NumSubRegs > 1)
836         MIB.addReg(SuperReg, RegState::ImplicitDefine);
837     } else {
838       if (OnlyToVGPR)
839         return false;
840 
841       // Restore SGPR from a stack slot.
842       // FIXME: We should use S_LOAD_DWORD here for VI.
843       unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
844       unsigned Align = FrameInfo.getObjectAlignment(Index);
845 
846       MachinePointerInfo PtrInfo
847         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
848 
849       MachineMemOperand *MMO = MF->getMachineMemOperand(PtrInfo,
850         MachineMemOperand::MOLoad, EltSize,
851         MinAlign(Align, EltSize * i));
852 
853       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpReg)
854         .addFrameIndex(Index)                   // vaddr
855         .addReg(MFI->getScratchRSrcReg())       // srsrc
856         .addReg(MFI->getScratchWaveOffsetReg()) // soffset
857         .addImm(i * 4)                          // offset
858         .addMemOperand(MMO);
859 
860       auto MIB =
861         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg)
862         .addReg(TmpReg, RegState::Kill);
863 
864       if (NumSubRegs > 1)
865         MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine);
866     }
867   }
868 
869   if (M0CopyReg != AMDGPU::NoRegister) {
870     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
871       .addReg(M0CopyReg, RegState::Kill);
872   }
873 
874   MI->eraseFromParent();
875   return true;
876 }
877 
878 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to
879 /// a VGPR and the stack slot can be safely eliminated when all other users are
880 /// handled.
881 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex(
882   MachineBasicBlock::iterator MI,
883   int FI,
884   RegScavenger *RS) const {
885   switch (MI->getOpcode()) {
886   case AMDGPU::SI_SPILL_S512_SAVE:
887   case AMDGPU::SI_SPILL_S256_SAVE:
888   case AMDGPU::SI_SPILL_S128_SAVE:
889   case AMDGPU::SI_SPILL_S64_SAVE:
890   case AMDGPU::SI_SPILL_S32_SAVE:
891     return spillSGPR(MI, FI, RS, true);
892   case AMDGPU::SI_SPILL_S512_RESTORE:
893   case AMDGPU::SI_SPILL_S256_RESTORE:
894   case AMDGPU::SI_SPILL_S128_RESTORE:
895   case AMDGPU::SI_SPILL_S64_RESTORE:
896   case AMDGPU::SI_SPILL_S32_RESTORE:
897     return restoreSGPR(MI, FI, RS, true);
898   default:
899     llvm_unreachable("not an SGPR spill instruction");
900   }
901 }
902 
903 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
904                                         int SPAdj, unsigned FIOperandNum,
905                                         RegScavenger *RS) const {
906   MachineFunction *MF = MI->getParent()->getParent();
907   MachineRegisterInfo &MRI = MF->getRegInfo();
908   MachineBasicBlock *MBB = MI->getParent();
909   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
910   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
911   const SISubtarget &ST =  MF->getSubtarget<SISubtarget>();
912   const SIInstrInfo *TII = ST.getInstrInfo();
913   DebugLoc DL = MI->getDebugLoc();
914 
915   MachineOperand &FIOp = MI->getOperand(FIOperandNum);
916   int Index = MI->getOperand(FIOperandNum).getIndex();
917 
918   switch (MI->getOpcode()) {
919     // SGPR register spill
920     case AMDGPU::SI_SPILL_S512_SAVE:
921     case AMDGPU::SI_SPILL_S256_SAVE:
922     case AMDGPU::SI_SPILL_S128_SAVE:
923     case AMDGPU::SI_SPILL_S64_SAVE:
924     case AMDGPU::SI_SPILL_S32_SAVE: {
925       spillSGPR(MI, Index, RS);
926       break;
927     }
928 
929     // SGPR register restore
930     case AMDGPU::SI_SPILL_S512_RESTORE:
931     case AMDGPU::SI_SPILL_S256_RESTORE:
932     case AMDGPU::SI_SPILL_S128_RESTORE:
933     case AMDGPU::SI_SPILL_S64_RESTORE:
934     case AMDGPU::SI_SPILL_S32_RESTORE: {
935       restoreSGPR(MI, Index, RS);
936       break;
937     }
938 
939     // VGPR register spill
940     case AMDGPU::SI_SPILL_V512_SAVE:
941     case AMDGPU::SI_SPILL_V256_SAVE:
942     case AMDGPU::SI_SPILL_V128_SAVE:
943     case AMDGPU::SI_SPILL_V96_SAVE:
944     case AMDGPU::SI_SPILL_V64_SAVE:
945     case AMDGPU::SI_SPILL_V32_SAVE: {
946       const MachineOperand *VData = TII->getNamedOperand(*MI,
947                                                          AMDGPU::OpName::vdata);
948       buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET,
949             Index,
950             VData->getReg(), VData->isKill(),
951             TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(),
952             TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg(),
953             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
954             *MI->memoperands_begin(),
955             RS);
956       MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode()));
957       MI->eraseFromParent();
958       break;
959     }
960     case AMDGPU::SI_SPILL_V32_RESTORE:
961     case AMDGPU::SI_SPILL_V64_RESTORE:
962     case AMDGPU::SI_SPILL_V96_RESTORE:
963     case AMDGPU::SI_SPILL_V128_RESTORE:
964     case AMDGPU::SI_SPILL_V256_RESTORE:
965     case AMDGPU::SI_SPILL_V512_RESTORE: {
966       const MachineOperand *VData = TII->getNamedOperand(*MI,
967                                                          AMDGPU::OpName::vdata);
968 
969       buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET,
970             Index,
971             VData->getReg(), VData->isKill(),
972             TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(),
973             TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg(),
974             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
975             *MI->memoperands_begin(),
976             RS);
977       MI->eraseFromParent();
978       break;
979     }
980 
981     default: {
982       if (TII->isMUBUF(*MI)) {
983         // Disable offen so we don't need a 0 vgpr base.
984         assert(static_cast<int>(FIOperandNum) ==
985                AMDGPU::getNamedOperandIdx(MI->getOpcode(),
986                                           AMDGPU::OpName::vaddr));
987 
988         int64_t Offset = FrameInfo.getObjectOffset(Index);
989         int64_t OldImm
990           = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm();
991         int64_t NewOffset = OldImm + Offset;
992 
993         if (isUInt<12>(NewOffset) &&
994             buildMUBUFOffsetLoadStore(TII, FrameInfo, MI, Index, NewOffset)) {
995           MI->eraseFromParent();
996           break;
997         }
998       }
999 
1000       int64_t Offset = FrameInfo.getObjectOffset(Index);
1001       FIOp.ChangeToImmediate(Offset);
1002       if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) {
1003         unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1004         BuildMI(*MBB, MI, MI->getDebugLoc(),
1005                 TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
1006                 .addImm(Offset);
1007         FIOp.ChangeToRegister(TmpReg, false, false, true);
1008       }
1009     }
1010   }
1011 }
1012 
1013 // FIXME: This is very slow. It might be worth creating a map from physreg to
1014 // register class.
1015 const TargetRegisterClass *SIRegisterInfo::getPhysRegClass(unsigned Reg) const {
1016   assert(!TargetRegisterInfo::isVirtualRegister(Reg));
1017 
1018   static const TargetRegisterClass *const BaseClasses[] = {
1019     &AMDGPU::VGPR_32RegClass,
1020     &AMDGPU::SReg_32RegClass,
1021     &AMDGPU::VReg_64RegClass,
1022     &AMDGPU::SReg_64RegClass,
1023     &AMDGPU::VReg_96RegClass,
1024     &AMDGPU::VReg_128RegClass,
1025     &AMDGPU::SReg_128RegClass,
1026     &AMDGPU::VReg_256RegClass,
1027     &AMDGPU::SReg_256RegClass,
1028     &AMDGPU::VReg_512RegClass,
1029     &AMDGPU::SReg_512RegClass,
1030     &AMDGPU::SCC_CLASSRegClass,
1031   };
1032 
1033   for (const TargetRegisterClass *BaseClass : BaseClasses) {
1034     if (BaseClass->contains(Reg)) {
1035       return BaseClass;
1036     }
1037   }
1038   return nullptr;
1039 }
1040 
1041 // TODO: It might be helpful to have some target specific flags in
1042 // TargetRegisterClass to mark which classes are VGPRs to make this trivial.
1043 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const {
1044   switch (RC->getSize()) {
1045   case 0: return false;
1046   case 1: return false;
1047   case 4:
1048     return getCommonSubClass(&AMDGPU::VGPR_32RegClass, RC) != nullptr;
1049   case 8:
1050     return getCommonSubClass(&AMDGPU::VReg_64RegClass, RC) != nullptr;
1051   case 12:
1052     return getCommonSubClass(&AMDGPU::VReg_96RegClass, RC) != nullptr;
1053   case 16:
1054     return getCommonSubClass(&AMDGPU::VReg_128RegClass, RC) != nullptr;
1055   case 32:
1056     return getCommonSubClass(&AMDGPU::VReg_256RegClass, RC) != nullptr;
1057   case 64:
1058     return getCommonSubClass(&AMDGPU::VReg_512RegClass, RC) != nullptr;
1059   default:
1060     llvm_unreachable("Invalid register class size");
1061   }
1062 }
1063 
1064 const TargetRegisterClass *SIRegisterInfo::getEquivalentVGPRClass(
1065                                          const TargetRegisterClass *SRC) const {
1066   switch (SRC->getSize()) {
1067   case 4:
1068     return &AMDGPU::VGPR_32RegClass;
1069   case 8:
1070     return &AMDGPU::VReg_64RegClass;
1071   case 12:
1072     return &AMDGPU::VReg_96RegClass;
1073   case 16:
1074     return &AMDGPU::VReg_128RegClass;
1075   case 32:
1076     return &AMDGPU::VReg_256RegClass;
1077   case 64:
1078     return &AMDGPU::VReg_512RegClass;
1079   default:
1080     llvm_unreachable("Invalid register class size");
1081   }
1082 }
1083 
1084 const TargetRegisterClass *SIRegisterInfo::getEquivalentSGPRClass(
1085                                          const TargetRegisterClass *VRC) const {
1086   switch (VRC->getSize()) {
1087   case 4:
1088     return &AMDGPU::SGPR_32RegClass;
1089   case 8:
1090     return &AMDGPU::SReg_64RegClass;
1091   case 16:
1092     return &AMDGPU::SReg_128RegClass;
1093   case 32:
1094     return &AMDGPU::SReg_256RegClass;
1095   case 64:
1096     return &AMDGPU::SReg_512RegClass;
1097   default:
1098     llvm_unreachable("Invalid register class size");
1099   }
1100 }
1101 
1102 const TargetRegisterClass *SIRegisterInfo::getSubRegClass(
1103                          const TargetRegisterClass *RC, unsigned SubIdx) const {
1104   if (SubIdx == AMDGPU::NoSubRegister)
1105     return RC;
1106 
1107   // We can assume that each lane corresponds to one 32-bit register.
1108   LaneBitmask::Type Mask = getSubRegIndexLaneMask(SubIdx).getAsInteger();
1109   unsigned Count = countPopulation(Mask);
1110   if (isSGPRClass(RC)) {
1111     switch (Count) {
1112     case 1:
1113       return &AMDGPU::SGPR_32RegClass;
1114     case 2:
1115       return &AMDGPU::SReg_64RegClass;
1116     case 4:
1117       return &AMDGPU::SReg_128RegClass;
1118     case 8:
1119       return &AMDGPU::SReg_256RegClass;
1120     case 16: /* fall-through */
1121     default:
1122       llvm_unreachable("Invalid sub-register class size");
1123     }
1124   } else {
1125     switch (Count) {
1126     case 1:
1127       return &AMDGPU::VGPR_32RegClass;
1128     case 2:
1129       return &AMDGPU::VReg_64RegClass;
1130     case 3:
1131       return &AMDGPU::VReg_96RegClass;
1132     case 4:
1133       return &AMDGPU::VReg_128RegClass;
1134     case 8:
1135       return &AMDGPU::VReg_256RegClass;
1136     case 16: /* fall-through */
1137     default:
1138       llvm_unreachable("Invalid sub-register class size");
1139     }
1140   }
1141 }
1142 
1143 bool SIRegisterInfo::shouldRewriteCopySrc(
1144   const TargetRegisterClass *DefRC,
1145   unsigned DefSubReg,
1146   const TargetRegisterClass *SrcRC,
1147   unsigned SrcSubReg) const {
1148   // We want to prefer the smallest register class possible, so we don't want to
1149   // stop and rewrite on anything that looks like a subregister
1150   // extract. Operations mostly don't care about the super register class, so we
1151   // only want to stop on the most basic of copies between the same register
1152   // class.
1153   //
1154   // e.g. if we have something like
1155   // vreg0 = ...
1156   // vreg1 = ...
1157   // vreg2 = REG_SEQUENCE vreg0, sub0, vreg1, sub1, vreg2, sub2
1158   // vreg3 = COPY vreg2, sub0
1159   //
1160   // We want to look through the COPY to find:
1161   //  => vreg3 = COPY vreg0
1162 
1163   // Plain copy.
1164   return getCommonSubClass(DefRC, SrcRC) != nullptr;
1165 }
1166 
1167 // FIXME: Most of these are flexible with HSA and we don't need to reserve them
1168 // as input registers if unused. Whether the dispatch ptr is necessary should be
1169 // easy to detect from used intrinsics. Scratch setup is harder to know.
1170 unsigned SIRegisterInfo::getPreloadedValue(const MachineFunction &MF,
1171                                            enum PreloadedValue Value) const {
1172 
1173   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1174   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
1175   (void)ST;
1176   switch (Value) {
1177   case SIRegisterInfo::WORKGROUP_ID_X:
1178     assert(MFI->hasWorkGroupIDX());
1179     return MFI->WorkGroupIDXSystemSGPR;
1180   case SIRegisterInfo::WORKGROUP_ID_Y:
1181     assert(MFI->hasWorkGroupIDY());
1182     return MFI->WorkGroupIDYSystemSGPR;
1183   case SIRegisterInfo::WORKGROUP_ID_Z:
1184     assert(MFI->hasWorkGroupIDZ());
1185     return MFI->WorkGroupIDZSystemSGPR;
1186   case SIRegisterInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET:
1187     return MFI->PrivateSegmentWaveByteOffsetSystemSGPR;
1188   case SIRegisterInfo::PRIVATE_SEGMENT_BUFFER:
1189     if (ST.isAmdCodeObjectV2(MF)) {
1190       assert(MFI->hasPrivateSegmentBuffer());
1191       return MFI->PrivateSegmentBufferUserSGPR;
1192     }
1193     assert(MFI->hasPrivateMemoryInputPtr());
1194     return MFI->PrivateMemoryPtrUserSGPR;
1195   case SIRegisterInfo::KERNARG_SEGMENT_PTR:
1196     assert(MFI->hasKernargSegmentPtr());
1197     return MFI->KernargSegmentPtrUserSGPR;
1198   case SIRegisterInfo::DISPATCH_ID:
1199     assert(MFI->hasDispatchID());
1200     return MFI->DispatchIDUserSGPR;
1201   case SIRegisterInfo::FLAT_SCRATCH_INIT:
1202     assert(MFI->hasFlatScratchInit());
1203     return MFI->FlatScratchInitUserSGPR;
1204   case SIRegisterInfo::DISPATCH_PTR:
1205     assert(MFI->hasDispatchPtr());
1206     return MFI->DispatchPtrUserSGPR;
1207   case SIRegisterInfo::QUEUE_PTR:
1208     assert(MFI->hasQueuePtr());
1209     return MFI->QueuePtrUserSGPR;
1210   case SIRegisterInfo::WORKITEM_ID_X:
1211     assert(MFI->hasWorkItemIDX());
1212     return AMDGPU::VGPR0;
1213   case SIRegisterInfo::WORKITEM_ID_Y:
1214     assert(MFI->hasWorkItemIDY());
1215     return AMDGPU::VGPR1;
1216   case SIRegisterInfo::WORKITEM_ID_Z:
1217     assert(MFI->hasWorkItemIDZ());
1218     return AMDGPU::VGPR2;
1219   }
1220   llvm_unreachable("unexpected preloaded value type");
1221 }
1222 
1223 /// \brief Returns a register that is not used at any point in the function.
1224 ///        If all registers are used, then this function will return
1225 //         AMDGPU::NoRegister.
1226 unsigned
1227 SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
1228                                    const TargetRegisterClass *RC,
1229                                    const MachineFunction &MF) const {
1230 
1231   for (unsigned Reg : *RC)
1232     if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
1233       return Reg;
1234   return AMDGPU::NoRegister;
1235 }
1236 
1237 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC,
1238                                                    unsigned EltSize) const {
1239   if (EltSize == 4) {
1240     static const int16_t Sub0_15[] = {
1241       AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1242       AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1243       AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1244       AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1245     };
1246 
1247     static const int16_t Sub0_7[] = {
1248       AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1249       AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1250     };
1251 
1252     static const int16_t Sub0_3[] = {
1253       AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1254     };
1255 
1256     static const int16_t Sub0_2[] = {
1257       AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2,
1258     };
1259 
1260     static const int16_t Sub0_1[] = {
1261       AMDGPU::sub0, AMDGPU::sub1,
1262     };
1263 
1264     switch (AMDGPU::getRegBitWidth(*RC->MC)) {
1265     case 32:
1266       return {};
1267     case 64:
1268       return makeArrayRef(Sub0_1);
1269     case 96:
1270       return makeArrayRef(Sub0_2);
1271     case 128:
1272       return makeArrayRef(Sub0_3);
1273     case 256:
1274       return makeArrayRef(Sub0_7);
1275     case 512:
1276       return makeArrayRef(Sub0_15);
1277     default:
1278       llvm_unreachable("unhandled register size");
1279     }
1280   }
1281 
1282   if (EltSize == 8) {
1283     static const int16_t Sub0_15_64[] = {
1284       AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1285       AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1286       AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1287       AMDGPU::sub12_sub13, AMDGPU::sub14_sub15
1288     };
1289 
1290     static const int16_t Sub0_7_64[] = {
1291       AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1292       AMDGPU::sub4_sub5, AMDGPU::sub6_sub7
1293     };
1294 
1295 
1296     static const int16_t Sub0_3_64[] = {
1297       AMDGPU::sub0_sub1, AMDGPU::sub2_sub3
1298     };
1299 
1300     switch (AMDGPU::getRegBitWidth(*RC->MC)) {
1301     case 64:
1302       return {};
1303     case 128:
1304       return makeArrayRef(Sub0_3_64);
1305     case 256:
1306       return makeArrayRef(Sub0_7_64);
1307     case 512:
1308       return makeArrayRef(Sub0_15_64);
1309     default:
1310       llvm_unreachable("unhandled register size");
1311     }
1312   }
1313 
1314   assert(EltSize == 16 && "unhandled register spill split size");
1315 
1316   static const int16_t Sub0_15_128[] = {
1317     AMDGPU::sub0_sub1_sub2_sub3,
1318     AMDGPU::sub4_sub5_sub6_sub7,
1319     AMDGPU::sub8_sub9_sub10_sub11,
1320     AMDGPU::sub12_sub13_sub14_sub15
1321   };
1322 
1323   static const int16_t Sub0_7_128[] = {
1324     AMDGPU::sub0_sub1_sub2_sub3,
1325     AMDGPU::sub4_sub5_sub6_sub7
1326   };
1327 
1328   switch (AMDGPU::getRegBitWidth(*RC->MC)) {
1329   case 128:
1330     return {};
1331   case 256:
1332     return makeArrayRef(Sub0_7_128);
1333   case 512:
1334     return makeArrayRef(Sub0_15_128);
1335   default:
1336     llvm_unreachable("unhandled register size");
1337   }
1338 }
1339 
1340 const TargetRegisterClass*
1341 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI,
1342                                   unsigned Reg) const {
1343   if (TargetRegisterInfo::isVirtualRegister(Reg))
1344     return  MRI.getRegClass(Reg);
1345 
1346   return getPhysRegClass(Reg);
1347 }
1348 
1349 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI,
1350                             unsigned Reg) const {
1351   return hasVGPRs(getRegClassForReg(MRI, Reg));
1352 }
1353 
1354 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI,
1355                                     const TargetRegisterClass *SrcRC,
1356                                     unsigned SubReg,
1357                                     const TargetRegisterClass *DstRC,
1358                                     unsigned DstSubReg,
1359                                     const TargetRegisterClass *NewRC) const {
1360   unsigned SrcSize = SrcRC->getSize();
1361   unsigned DstSize = DstRC->getSize();
1362   unsigned NewSize = NewRC->getSize();
1363 
1364   // Do not increase size of registers beyond dword, we would need to allocate
1365   // adjacent registers and constraint regalloc more than needed.
1366 
1367   // Always allow dword coalescing.
1368   if (SrcSize <= 4 || DstSize <= 4)
1369     return true;
1370 
1371   return NewSize <= DstSize || NewSize <= SrcSize;
1372 }
1373 
1374 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
1375                                              MachineFunction &MF) const {
1376 
1377   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
1378   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1379 
1380   unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
1381                                                        *MF.getFunction());
1382   switch (RC->getID()) {
1383   default:
1384     return AMDGPURegisterInfo::getRegPressureLimit(RC, MF);
1385   case AMDGPU::VGPR_32RegClassID:
1386     return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF));
1387   case AMDGPU::SGPR_32RegClassID:
1388     return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF));
1389   }
1390 }
1391 
1392 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF,
1393                                                 unsigned Idx) const {
1394   if (Idx == getVGPRPressureSet())
1395     return getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
1396                                const_cast<MachineFunction &>(MF));
1397 
1398   if (Idx == getSGPRPressureSet())
1399     return getRegPressureLimit(&AMDGPU::SGPR_32RegClass,
1400                                const_cast<MachineFunction &>(MF));
1401 
1402   return AMDGPURegisterInfo::getRegPressureSetLimit(MF, Idx);
1403 }
1404 
1405 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const {
1406   static const int Empty[] = { -1 };
1407 
1408   if (hasRegUnit(AMDGPU::M0, RegUnit))
1409     return Empty;
1410   return AMDGPURegisterInfo::getRegUnitPressureSets(RegUnit);
1411 }
1412 
1413 unsigned SIRegisterInfo::getRegUnitWeight(const MachineRegisterInfo &MRI,
1414                                           unsigned RegUnit,
1415                                           LaneBitmask LaneMask) const {
1416   unsigned Weight = TargetRegisterInfo::getRegUnitWeight(MRI, RegUnit,
1417                                                          LaneMask);
1418   if (Weight > 1 && LaneMask.any() && !LaneMask.all() &&
1419       isVirtualRegister(RegUnit)) {
1420     LaneBitmask Max = MRI.getMaxLaneMaskForVReg(RegUnit);
1421     if (Max != LaneMask && !Max.all() && !Max.none())
1422       Weight = (Weight * countPopulation(LaneMask.getAsInteger())) /
1423                          countPopulation(Max.getAsInteger());
1424   }
1425   return Weight;
1426 }
1427