1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI implementation of the TargetRegisterInfo class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIRegisterInfo.h"
15 #include "AMDGPURegisterBankInfo.h"
16 #include "AMDGPUSubtarget.h"
17 #include "SIInstrInfo.h"
18 #include "SIMachineFunctionInfo.h"
19 #include "MCTargetDesc/AMDGPUInstPrinter.h"
20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21 #include "llvm/CodeGen/LiveIntervals.h"
22 #include "llvm/CodeGen/MachineDominators.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/RegisterScavenging.h"
26 #include "llvm/CodeGen/SlotIndexes.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include <vector>
30 
31 using namespace llvm;
32 
33 #define GET_REGINFO_TARGET_DESC
34 #include "AMDGPUGenRegisterInfo.inc"
35 
36 static cl::opt<bool> EnableSpillSGPRToVGPR(
37   "amdgpu-spill-sgpr-to-vgpr",
38   cl::desc("Enable spilling VGPRs to SGPRs"),
39   cl::ReallyHidden,
40   cl::init(true));
41 
42 std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts;
43 std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable;
44 
45 // Map numbers of DWORDs to indexes in SubRegFromChannelTable.
46 // Valid indexes are shifted 1, such that a 0 mapping means unsupported.
47 // e.g. for 8 DWORDs (256-bit), SubRegFromChannelTableWidthMap[8] = 8,
48 //      meaning index 7 in SubRegFromChannelTable.
49 static const std::array<unsigned, 17> SubRegFromChannelTableWidthMap = {
50     0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9};
51 
52 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST)
53     : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST),
54       SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {
55 
56   assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
57          getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
58          (getSubRegIndexLaneMask(AMDGPU::lo16) |
59           getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
60            getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() &&
61          "getNumCoveredRegs() will not work with generated subreg masks!");
62 
63   RegPressureIgnoredUnits.resize(getNumRegUnits());
64   RegPressureIgnoredUnits.set(
65       *MCRegUnitIterator(MCRegister::from(AMDGPU::M0), this));
66   for (auto Reg : AMDGPU::VGPR_HI16RegClass)
67     RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this));
68 
69   // HACK: Until this is fully tablegen'd.
70   static llvm::once_flag InitializeRegSplitPartsFlag;
71 
72   static auto InitializeRegSplitPartsOnce = [this]() {
73     for (unsigned Idx = 1, E = getNumSubRegIndices() - 1; Idx < E; ++Idx) {
74       unsigned Size = getSubRegIdxSize(Idx);
75       if (Size & 31)
76         continue;
77       std::vector<int16_t> &Vec = RegSplitParts[Size / 32 - 1];
78       unsigned Pos = getSubRegIdxOffset(Idx);
79       if (Pos % Size)
80         continue;
81       Pos /= Size;
82       if (Vec.empty()) {
83         unsigned MaxNumParts = 1024 / Size; // Maximum register is 1024 bits.
84         Vec.resize(MaxNumParts);
85       }
86       Vec[Pos] = Idx;
87     }
88   };
89 
90   static llvm::once_flag InitializeSubRegFromChannelTableFlag;
91 
92   static auto InitializeSubRegFromChannelTableOnce = [this]() {
93     for (auto &Row : SubRegFromChannelTable)
94       Row.fill(AMDGPU::NoSubRegister);
95     for (uint16_t Idx = 1; Idx < getNumSubRegIndices(); ++Idx) {
96       unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32;
97       unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32;
98       assert(Width < SubRegFromChannelTableWidthMap.size());
99       Width = SubRegFromChannelTableWidthMap[Width];
100       if (Width == 0)
101         continue;
102       unsigned TableIdx = Width - 1;
103       assert(TableIdx < SubRegFromChannelTable.size());
104       assert(Offset < SubRegFromChannelTable[TableIdx].size());
105       SubRegFromChannelTable[TableIdx][Offset] = Idx;
106     }
107   };
108 
109   llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce);
110   llvm::call_once(InitializeSubRegFromChannelTableFlag,
111                   InitializeSubRegFromChannelTableOnce);
112 }
113 
114 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved,
115                                            MCRegister Reg) const {
116   MCRegAliasIterator R(Reg, this, true);
117 
118   for (; R.isValid(); ++R)
119     Reserved.set(*R);
120 }
121 
122 // Forced to be here by one .inc
123 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs(
124   const MachineFunction *MF) const {
125   CallingConv::ID CC = MF->getFunction().getCallingConv();
126   switch (CC) {
127   case CallingConv::C:
128   case CallingConv::Fast:
129   case CallingConv::Cold:
130   case CallingConv::AMDGPU_Gfx:
131     return CSR_AMDGPU_HighRegs_SaveList;
132   default: {
133     // Dummy to not crash RegisterClassInfo.
134     static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister;
135     return &NoCalleeSavedReg;
136   }
137   }
138 }
139 
140 const MCPhysReg *
141 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const {
142   return nullptr;
143 }
144 
145 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
146                                                      CallingConv::ID CC) const {
147   switch (CC) {
148   case CallingConv::C:
149   case CallingConv::Fast:
150   case CallingConv::Cold:
151   case CallingConv::AMDGPU_Gfx:
152     return CSR_AMDGPU_HighRegs_RegMask;
153   default:
154     return nullptr;
155   }
156 }
157 
158 const uint32_t *SIRegisterInfo::getNoPreservedMask() const {
159   return CSR_AMDGPU_NoRegs_RegMask;
160 }
161 
162 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
163   const SIFrameLowering *TFI =
164       MF.getSubtarget<GCNSubtarget>().getFrameLowering();
165   const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
166   // During ISel lowering we always reserve the stack pointer in entry
167   // functions, but never actually want to reference it when accessing our own
168   // frame. If we need a frame pointer we use it, but otherwise we can just use
169   // an immediate "0" which we represent by returning NoRegister.
170   if (FuncInfo->isEntryFunction()) {
171     return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register();
172   }
173   return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg()
174                         : FuncInfo->getStackPtrOffsetReg();
175 }
176 
177 bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const {
178   // When we need stack realignment, we can't reference off of the
179   // stack pointer, so we reserve a base pointer.
180   const MachineFrameInfo &MFI = MF.getFrameInfo();
181   return MFI.getNumFixedObjects() && needsStackRealignment(MF);
182 }
183 
184 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; }
185 
186 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const {
187   return CSR_AMDGPU_AllVGPRs_RegMask;
188 }
189 
190 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const {
191   return CSR_AMDGPU_AllAllocatableSRegs_RegMask;
192 }
193 
194 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
195                                               unsigned NumRegs) {
196   assert(NumRegs < SubRegFromChannelTableWidthMap.size());
197   unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs];
198   assert(NumRegIndex && "Not implemented");
199   assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].size());
200   return SubRegFromChannelTable[NumRegIndex - 1][Channel];
201 }
202 
203 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg(
204   const MachineFunction &MF) const {
205   unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
206   MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
207   return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass);
208 }
209 
210 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
211   BitVector Reserved(getNumRegs());
212   Reserved.set(AMDGPU::MODE);
213 
214   // EXEC_LO and EXEC_HI could be allocated and used as regular register, but
215   // this seems likely to result in bugs, so I'm marking them as reserved.
216   reserveRegisterTuples(Reserved, AMDGPU::EXEC);
217   reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR);
218 
219   // M0 has to be reserved so that llvm accepts it as a live-in into a block.
220   reserveRegisterTuples(Reserved, AMDGPU::M0);
221 
222   // Reserve src_vccz, src_execz, src_scc.
223   reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ);
224   reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ);
225   reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC);
226 
227   // Reserve the memory aperture registers.
228   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE);
229   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT);
230   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
231   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
232 
233   // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen.
234   reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID);
235 
236   // Reserve xnack_mask registers - support is not implemented in Codegen.
237   reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK);
238 
239   // Reserve lds_direct register - support is not implemented in Codegen.
240   reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT);
241 
242   // Reserve Trap Handler registers - support is not implemented in Codegen.
243   reserveRegisterTuples(Reserved, AMDGPU::TBA);
244   reserveRegisterTuples(Reserved, AMDGPU::TMA);
245   reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1);
246   reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3);
247   reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5);
248   reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7);
249   reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9);
250   reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11);
251   reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13);
252   reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15);
253 
254   // Reserve null register - it shall never be allocated
255   reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL);
256 
257   // Disallow vcc_hi allocation in wave32. It may be allocated but most likely
258   // will result in bugs.
259   if (isWave32) {
260     Reserved.set(AMDGPU::VCC);
261     Reserved.set(AMDGPU::VCC_HI);
262   }
263 
264   unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
265   unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
266   for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) {
267     unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
268     reserveRegisterTuples(Reserved, Reg);
269   }
270 
271   unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF);
272   unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
273   for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) {
274     unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i);
275     reserveRegisterTuples(Reserved, Reg);
276     Reg = AMDGPU::AGPR_32RegClass.getRegister(i);
277     reserveRegisterTuples(Reserved, Reg);
278   }
279 
280   for (auto Reg : AMDGPU::SReg_32RegClass) {
281     Reserved.set(getSubReg(Reg, AMDGPU::hi16));
282     Register Low = getSubReg(Reg, AMDGPU::lo16);
283     // This is to prevent BB vcc liveness errors.
284     if (!AMDGPU::SGPR_LO16RegClass.contains(Low))
285       Reserved.set(Low);
286   }
287 
288   for (auto Reg : AMDGPU::AGPR_32RegClass) {
289     Reserved.set(getSubReg(Reg, AMDGPU::hi16));
290   }
291 
292   // Reserve all the rest AGPRs if there are no instructions to use it.
293   if (!ST.hasMAIInsts()) {
294     for (unsigned i = 0; i < MaxNumVGPRs; ++i) {
295       unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i);
296       reserveRegisterTuples(Reserved, Reg);
297     }
298   }
299 
300   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
301 
302   Register ScratchRSrcReg = MFI->getScratchRSrcReg();
303   if (ScratchRSrcReg != AMDGPU::NoRegister) {
304     // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need
305     // to spill.
306     // TODO: May need to reserve a VGPR if doing LDS spilling.
307     reserveRegisterTuples(Reserved, ScratchRSrcReg);
308   }
309 
310   // We have to assume the SP is needed in case there are calls in the function,
311   // which is detected after the function is lowered. If we aren't really going
312   // to need SP, don't bother reserving it.
313   MCRegister StackPtrReg = MFI->getStackPtrOffsetReg();
314 
315   if (StackPtrReg) {
316     reserveRegisterTuples(Reserved, StackPtrReg);
317     assert(!isSubRegister(ScratchRSrcReg, StackPtrReg));
318   }
319 
320   MCRegister FrameReg = MFI->getFrameOffsetReg();
321   if (FrameReg) {
322     reserveRegisterTuples(Reserved, FrameReg);
323     assert(!isSubRegister(ScratchRSrcReg, FrameReg));
324   }
325 
326   if (hasBasePointer(MF)) {
327     MCRegister BasePtrReg = getBaseRegister();
328     reserveRegisterTuples(Reserved, BasePtrReg);
329     assert(!isSubRegister(ScratchRSrcReg, BasePtrReg));
330   }
331 
332   for (MCRegister Reg : MFI->WWMReservedRegs) {
333     reserveRegisterTuples(Reserved, Reg);
334   }
335 
336   // FIXME: Stop using reserved registers for this.
337   for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs())
338     reserveRegisterTuples(Reserved, Reg);
339 
340   for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs())
341     reserveRegisterTuples(Reserved, Reg);
342 
343   for (auto SSpill : MFI->getSGPRSpillVGPRs())
344     reserveRegisterTuples(Reserved, SSpill.VGPR);
345 
346   return Reserved;
347 }
348 
349 bool SIRegisterInfo::canRealignStack(const MachineFunction &MF) const {
350   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
351   // On entry, the base address is 0, so it can't possibly need any more
352   // alignment.
353 
354   // FIXME: Should be able to specify the entry frame alignment per calling
355   // convention instead.
356   if (Info->isEntryFunction())
357     return false;
358 
359   return TargetRegisterInfo::canRealignStack(MF);
360 }
361 
362 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const {
363   const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>();
364   if (Info->isEntryFunction()) {
365     const MachineFrameInfo &MFI = Fn.getFrameInfo();
366     return MFI.hasStackObjects() || MFI.hasCalls();
367   }
368 
369   // May need scavenger for dealing with callee saved registers.
370   return true;
371 }
372 
373 bool SIRegisterInfo::requiresFrameIndexScavenging(
374   const MachineFunction &MF) const {
375   // Do not use frame virtual registers. They used to be used for SGPRs, but
376   // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the
377   // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a
378   // spill.
379   return false;
380 }
381 
382 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging(
383   const MachineFunction &MF) const {
384   const MachineFrameInfo &MFI = MF.getFrameInfo();
385   return MFI.hasStackObjects();
386 }
387 
388 bool SIRegisterInfo::requiresVirtualBaseRegisters(
389   const MachineFunction &) const {
390   // There are no special dedicated stack or frame pointers.
391   return true;
392 }
393 
394 int64_t SIRegisterInfo::getScratchInstrOffset(const MachineInstr *MI) const {
395   assert(SIInstrInfo::isMUBUF(*MI) || SIInstrInfo::isFLATScratch(*MI));
396 
397   int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
398                                           AMDGPU::OpName::offset);
399   return MI->getOperand(OffIdx).getImm();
400 }
401 
402 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI,
403                                                  int Idx) const {
404   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI))
405     return 0;
406 
407   assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(),
408                                             AMDGPU::OpName::vaddr) ||
409          (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(),
410                                             AMDGPU::OpName::saddr))) &&
411          "Should never see frame index on non-address operand");
412 
413   return getScratchInstrOffset(MI);
414 }
415 
416 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
417   if (!MI->mayLoadOrStore())
418     return false;
419 
420   int64_t FullOffset = Offset + getScratchInstrOffset(MI);
421 
422   if (SIInstrInfo::isMUBUF(*MI))
423     return !SIInstrInfo::isLegalMUBUFImmOffset(FullOffset);
424 
425   const SIInstrInfo *TII = ST.getInstrInfo();
426   return TII->isLegalFLATOffset(FullOffset, AMDGPUAS::PRIVATE_ADDRESS, true);
427 }
428 
429 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
430                                                   Register BaseReg,
431                                                   int FrameIdx,
432                                                   int64_t Offset) const {
433   MachineBasicBlock::iterator Ins = MBB->begin();
434   DebugLoc DL; // Defaults to "unknown"
435 
436   if (Ins != MBB->end())
437     DL = Ins->getDebugLoc();
438 
439   MachineFunction *MF = MBB->getParent();
440   const SIInstrInfo *TII = ST.getInstrInfo();
441   unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32
442                                            : AMDGPU::V_MOV_B32_e32;
443 
444   if (Offset == 0) {
445     BuildMI(*MBB, Ins, DL, TII->get(MovOpc), BaseReg)
446       .addFrameIndex(FrameIdx);
447     return;
448   }
449 
450   MachineRegisterInfo &MRI = MF->getRegInfo();
451   Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
452 
453   Register FIReg = MRI.createVirtualRegister(
454       ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass
455                              : &AMDGPU::VGPR_32RegClass);
456 
457   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
458     .addImm(Offset);
459   BuildMI(*MBB, Ins, DL, TII->get(MovOpc), FIReg)
460     .addFrameIndex(FrameIdx);
461 
462   if (ST.enableFlatScratch() ) {
463     BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_U32), BaseReg)
464         .addReg(OffsetReg, RegState::Kill)
465         .addReg(FIReg);
466     return;
467   }
468 
469   TII->getAddNoCarry(*MBB, Ins, DL, BaseReg)
470     .addReg(OffsetReg, RegState::Kill)
471     .addReg(FIReg)
472     .addImm(0); // clamp bit
473 }
474 
475 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
476                                        int64_t Offset) const {
477   const SIInstrInfo *TII = ST.getInstrInfo();
478   bool IsFlat = TII->isFLATScratch(MI);
479 
480 #ifndef NDEBUG
481   // FIXME: Is it possible to be storing a frame index to itself?
482   bool SeenFI = false;
483   for (const MachineOperand &MO: MI.operands()) {
484     if (MO.isFI()) {
485       if (SeenFI)
486         llvm_unreachable("should not see multiple frame indices");
487 
488       SeenFI = true;
489     }
490   }
491 #endif
492 
493   MachineOperand *FIOp =
494       TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr
495                                       : AMDGPU::OpName::vaddr);
496 
497   MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset);
498   int64_t NewOffset = OffsetOp->getImm() + Offset;
499 
500 #ifndef NDEBUG
501   MachineBasicBlock *MBB = MI.getParent();
502   MachineFunction *MF = MBB->getParent();
503   assert(FIOp && FIOp->isFI() && "frame index must be address operand");
504   assert(TII->isMUBUF(MI) || TII->isFLATScratch(MI));
505 
506   if (IsFlat) {
507     assert(TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, true) &&
508            "offset should be legal");
509     FIOp->ChangeToRegister(BaseReg, false);
510     OffsetOp->setImm(NewOffset);
511     return;
512   }
513 
514   MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
515   assert((SOffset->isReg() &&
516           SOffset->getReg() ==
517               MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg()) ||
518          (SOffset->isImm() && SOffset->getImm() == 0));
519 #endif
520 
521   assert(SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) &&
522          "offset should be legal");
523 
524   FIOp->ChangeToRegister(BaseReg, false);
525   OffsetOp->setImm(NewOffset);
526 }
527 
528 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
529                                         Register BaseReg,
530                                         int64_t Offset) const {
531   if (!SIInstrInfo::isMUBUF(*MI) && !!SIInstrInfo::isFLATScratch(*MI))
532     return false;
533 
534   int64_t NewOffset = Offset + getScratchInstrOffset(MI);
535 
536   if (SIInstrInfo::isMUBUF(*MI))
537     return SIInstrInfo::isLegalMUBUFImmOffset(NewOffset);
538 
539   const SIInstrInfo *TII = ST.getInstrInfo();
540   return TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, true);
541 }
542 
543 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass(
544   const MachineFunction &MF, unsigned Kind) const {
545   // This is inaccurate. It depends on the instruction and address space. The
546   // only place where we should hit this is for dealing with frame indexes /
547   // private accesses, so this is correct in that case.
548   return &AMDGPU::VGPR_32RegClass;
549 }
550 
551 static unsigned getNumSubRegsForSpillOp(unsigned Op) {
552 
553   switch (Op) {
554   case AMDGPU::SI_SPILL_S1024_SAVE:
555   case AMDGPU::SI_SPILL_S1024_RESTORE:
556   case AMDGPU::SI_SPILL_V1024_SAVE:
557   case AMDGPU::SI_SPILL_V1024_RESTORE:
558   case AMDGPU::SI_SPILL_A1024_SAVE:
559   case AMDGPU::SI_SPILL_A1024_RESTORE:
560     return 32;
561   case AMDGPU::SI_SPILL_S512_SAVE:
562   case AMDGPU::SI_SPILL_S512_RESTORE:
563   case AMDGPU::SI_SPILL_V512_SAVE:
564   case AMDGPU::SI_SPILL_V512_RESTORE:
565   case AMDGPU::SI_SPILL_A512_SAVE:
566   case AMDGPU::SI_SPILL_A512_RESTORE:
567     return 16;
568   case AMDGPU::SI_SPILL_S256_SAVE:
569   case AMDGPU::SI_SPILL_S256_RESTORE:
570   case AMDGPU::SI_SPILL_V256_SAVE:
571   case AMDGPU::SI_SPILL_V256_RESTORE:
572   case AMDGPU::SI_SPILL_A256_SAVE:
573   case AMDGPU::SI_SPILL_A256_RESTORE:
574     return 8;
575   case AMDGPU::SI_SPILL_S192_SAVE:
576   case AMDGPU::SI_SPILL_S192_RESTORE:
577   case AMDGPU::SI_SPILL_V192_SAVE:
578   case AMDGPU::SI_SPILL_V192_RESTORE:
579   case AMDGPU::SI_SPILL_A192_SAVE:
580   case AMDGPU::SI_SPILL_A192_RESTORE:
581     return 6;
582   case AMDGPU::SI_SPILL_S160_SAVE:
583   case AMDGPU::SI_SPILL_S160_RESTORE:
584   case AMDGPU::SI_SPILL_V160_SAVE:
585   case AMDGPU::SI_SPILL_V160_RESTORE:
586   case AMDGPU::SI_SPILL_A160_SAVE:
587   case AMDGPU::SI_SPILL_A160_RESTORE:
588     return 5;
589   case AMDGPU::SI_SPILL_S128_SAVE:
590   case AMDGPU::SI_SPILL_S128_RESTORE:
591   case AMDGPU::SI_SPILL_V128_SAVE:
592   case AMDGPU::SI_SPILL_V128_RESTORE:
593   case AMDGPU::SI_SPILL_A128_SAVE:
594   case AMDGPU::SI_SPILL_A128_RESTORE:
595     return 4;
596   case AMDGPU::SI_SPILL_S96_SAVE:
597   case AMDGPU::SI_SPILL_S96_RESTORE:
598   case AMDGPU::SI_SPILL_V96_SAVE:
599   case AMDGPU::SI_SPILL_V96_RESTORE:
600   case AMDGPU::SI_SPILL_A96_SAVE:
601   case AMDGPU::SI_SPILL_A96_RESTORE:
602     return 3;
603   case AMDGPU::SI_SPILL_S64_SAVE:
604   case AMDGPU::SI_SPILL_S64_RESTORE:
605   case AMDGPU::SI_SPILL_V64_SAVE:
606   case AMDGPU::SI_SPILL_V64_RESTORE:
607   case AMDGPU::SI_SPILL_A64_SAVE:
608   case AMDGPU::SI_SPILL_A64_RESTORE:
609     return 2;
610   case AMDGPU::SI_SPILL_S32_SAVE:
611   case AMDGPU::SI_SPILL_S32_RESTORE:
612   case AMDGPU::SI_SPILL_V32_SAVE:
613   case AMDGPU::SI_SPILL_V32_RESTORE:
614   case AMDGPU::SI_SPILL_A32_SAVE:
615   case AMDGPU::SI_SPILL_A32_RESTORE:
616     return 1;
617   default: llvm_unreachable("Invalid spill opcode");
618   }
619 }
620 
621 static int getOffsetMUBUFStore(unsigned Opc) {
622   switch (Opc) {
623   case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
624     return AMDGPU::BUFFER_STORE_DWORD_OFFSET;
625   case AMDGPU::BUFFER_STORE_BYTE_OFFEN:
626     return AMDGPU::BUFFER_STORE_BYTE_OFFSET;
627   case AMDGPU::BUFFER_STORE_SHORT_OFFEN:
628     return AMDGPU::BUFFER_STORE_SHORT_OFFSET;
629   case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN:
630     return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET;
631   case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN:
632     return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET;
633   case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN:
634     return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET;
635   case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN:
636     return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET;
637   default:
638     return -1;
639   }
640 }
641 
642 static int getOffsetMUBUFLoad(unsigned Opc) {
643   switch (Opc) {
644   case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
645     return AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
646   case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN:
647     return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET;
648   case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN:
649     return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET;
650   case AMDGPU::BUFFER_LOAD_USHORT_OFFEN:
651     return AMDGPU::BUFFER_LOAD_USHORT_OFFSET;
652   case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN:
653     return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET;
654   case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN:
655     return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET;
656   case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN:
657     return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET;
658   case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN:
659     return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET;
660   case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN:
661     return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET;
662   case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN:
663     return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET;
664   case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN:
665     return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET;
666   case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN:
667     return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET;
668   case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN:
669     return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET;
670   default:
671     return -1;
672   }
673 }
674 
675 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST,
676                                            MachineBasicBlock::iterator MI,
677                                            int Index,
678                                            unsigned Lane,
679                                            unsigned ValueReg,
680                                            bool IsKill) {
681   MachineBasicBlock *MBB = MI->getParent();
682   MachineFunction *MF = MI->getParent()->getParent();
683   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
684   const SIInstrInfo *TII = ST.getInstrInfo();
685 
686   MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane);
687 
688   if (Reg == AMDGPU::NoRegister)
689     return MachineInstrBuilder();
690 
691   bool IsStore = MI->mayStore();
692   MachineRegisterInfo &MRI = MF->getRegInfo();
693   auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
694 
695   unsigned Dst = IsStore ? Reg : ValueReg;
696   unsigned Src = IsStore ? ValueReg : Reg;
697   unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32
698                                                    : AMDGPU::V_ACCVGPR_READ_B32;
699 
700   auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst)
701                .addReg(Src, getKillRegState(IsKill));
702   MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
703   return MIB;
704 }
705 
706 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not
707 // need to handle the case where an SGPR may need to be spilled while spilling.
708 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST,
709                                       MachineFrameInfo &MFI,
710                                       MachineBasicBlock::iterator MI,
711                                       int Index,
712                                       int64_t Offset) {
713   const SIInstrInfo *TII = ST.getInstrInfo();
714   MachineBasicBlock *MBB = MI->getParent();
715   const DebugLoc &DL = MI->getDebugLoc();
716   bool IsStore = MI->mayStore();
717 
718   unsigned Opc = MI->getOpcode();
719   int LoadStoreOp = IsStore ?
720     getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc);
721   if (LoadStoreOp == -1)
722     return false;
723 
724   const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata);
725   if (spillVGPRtoAGPR(ST, MI, Index, 0, Reg->getReg(), false).getInstr())
726     return true;
727 
728   MachineInstrBuilder NewMI =
729       BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp))
730           .add(*Reg)
731           .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc))
732           .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset))
733           .addImm(Offset)
734           .addImm(0) // glc
735           .addImm(0) // slc
736           .addImm(0) // tfe
737           .addImm(0) // dlc
738           .addImm(0) // swz
739           .cloneMemRefs(*MI);
740 
741   const MachineOperand *VDataIn = TII->getNamedOperand(*MI,
742                                                        AMDGPU::OpName::vdata_in);
743   if (VDataIn)
744     NewMI.add(*VDataIn);
745   return true;
746 }
747 
748 static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII,
749                                           unsigned LoadStoreOp,
750                                           unsigned EltSize) {
751   bool IsStore = TII->get(LoadStoreOp).mayStore();
752   bool UseST =
753     AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0 &&
754     AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::saddr) < 0;
755 
756   switch (EltSize) {
757   case 4:
758     LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
759                           : AMDGPU::SCRATCH_LOAD_DWORD_SADDR;
760     break;
761   case 8:
762     LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR
763                           : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR;
764     break;
765   case 12:
766     LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR
767                           : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR;
768     break;
769   case 16:
770     LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR
771                           : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR;
772     break;
773   default:
774     llvm_unreachable("Unexpected spill load/store size!");
775   }
776 
777   if (UseST)
778     LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp);
779 
780   return LoadStoreOp;
781 }
782 
783 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
784                                          unsigned LoadStoreOp,
785                                          int Index,
786                                          Register ValueReg,
787                                          bool IsKill,
788                                          MCRegister ScratchOffsetReg,
789                                          int64_t InstOffset,
790                                          MachineMemOperand *MMO,
791                                          RegScavenger *RS) const {
792   MachineBasicBlock *MBB = MI->getParent();
793   MachineFunction *MF = MI->getParent()->getParent();
794   const SIInstrInfo *TII = ST.getInstrInfo();
795   const MachineFrameInfo &MFI = MF->getFrameInfo();
796   const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
797 
798   const MCInstrDesc *Desc = &TII->get(LoadStoreOp);
799   const DebugLoc &DL = MI->getDebugLoc();
800   bool IsStore = Desc->mayStore();
801   bool IsFlat = TII->isFLATScratch(LoadStoreOp);
802 
803   bool Scavenged = false;
804   MCRegister SOffset = ScratchOffsetReg;
805 
806   const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg);
807   const bool IsAGPR = hasAGPRs(RC);
808   const unsigned RegWidth = AMDGPU::getRegBitWidth(RC->getID()) / 8;
809 
810   // Always use 4 byte operations for AGPRs because we need to scavenge
811   // a temporary VGPR.
812   unsigned EltSize = (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u) : 4u;
813   unsigned NumSubRegs = RegWidth / EltSize;
814   unsigned Size = NumSubRegs * EltSize;
815   unsigned RemSize = RegWidth - Size;
816   unsigned NumRemSubRegs = RemSize ? 1 : 0;
817   int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
818   int64_t MaxOffset = Offset + Size + RemSize - EltSize;
819   int64_t ScratchOffsetRegDelta = 0;
820 
821   if (IsFlat && EltSize > 4) {
822     LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
823     Desc = &TII->get(LoadStoreOp);
824   }
825 
826   Align Alignment = MFI.getObjectAlign(Index);
827   const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo();
828 
829   assert((IsFlat || ((Offset % EltSize) == 0)) &&
830          "unexpected VGPR spill offset");
831 
832   bool IsOffsetLegal = IsFlat
833       ? TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS, true)
834       : SIInstrInfo::isLegalMUBUFImmOffset(MaxOffset);
835   if (!IsOffsetLegal || (IsFlat && !SOffset && !ST.hasFlatScratchSTMode())) {
836     SOffset = MCRegister();
837 
838     // We currently only support spilling VGPRs to EltSize boundaries, meaning
839     // we can simplify the adjustment of Offset here to just scale with
840     // WavefrontSize.
841     if (!IsFlat)
842       Offset *= ST.getWavefrontSize();
843 
844     // We don't have access to the register scavenger if this function is called
845     // during  PEI::scavengeFrameVirtualRegs().
846     if (RS)
847       SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false);
848 
849     if (!SOffset) {
850       // There are no free SGPRs, and since we are in the process of spilling
851       // VGPRs too.  Since we need a VGPR in order to spill SGPRs (this is true
852       // on SI/CI and on VI it is true until we implement spilling using scalar
853       // stores), we have no way to free up an SGPR.  Our solution here is to
854       // add the offset directly to the ScratchOffset or StackPtrOffset
855       // register, and then subtract the offset after the spill to return the
856       // register to it's original value.
857       if (!ScratchOffsetReg)
858         ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg();
859       SOffset = ScratchOffsetReg;
860       ScratchOffsetRegDelta = Offset;
861     } else {
862       Scavenged = true;
863     }
864 
865     if (!SOffset)
866       report_fatal_error("could not scavenge SGPR to spill in entry function");
867 
868     if (ScratchOffsetReg == AMDGPU::NoRegister) {
869       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset)
870           .addImm(Offset);
871     } else {
872       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset)
873           .addReg(ScratchOffsetReg)
874           .addImm(Offset);
875     }
876 
877     Offset = 0;
878   }
879 
880   if (IsFlat && SOffset == AMDGPU::NoRegister) {
881     assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0
882            && "Unexpected vaddr for flat scratch with a FI operand");
883 
884     assert(ST.hasFlatScratchSTMode());
885     LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp);
886     Desc = &TII->get(LoadStoreOp);
887   }
888 
889   Register TmpReg;
890 
891   for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
892        ++i, RegOffset += EltSize) {
893     if (i == NumSubRegs) {
894       EltSize = RemSize;
895       LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
896     }
897     Desc = &TII->get(LoadStoreOp);
898 
899     unsigned NumRegs = EltSize / 4;
900     Register SubReg = e == 1
901             ? ValueReg
902             : Register(getSubReg(ValueReg,
903                                  getSubRegFromChannel(RegOffset / 4, NumRegs)));
904 
905     unsigned SOffsetRegState = 0;
906     unsigned SrcDstRegState = getDefRegState(!IsStore);
907     if (i + 1 == e) {
908       SOffsetRegState |= getKillRegState(Scavenged);
909       // The last implicit use carries the "Kill" flag.
910       SrcDstRegState |= getKillRegState(IsKill);
911     }
912 
913     // Make sure the whole register is defined if there are undef components by
914     // adding an implicit def of the super-reg on the first instruction.
915     bool NeedSuperRegDef = e > 1 && IsStore && i == 0;
916     bool NeedSuperRegImpOperand = e > 1;
917 
918     unsigned Lane = RegOffset / 4;
919     unsigned LaneE = (RegOffset + EltSize) / 4;
920     for ( ; Lane != LaneE; ++Lane) {
921       bool IsSubReg = e > 1 || EltSize > 4;
922       Register Sub = IsSubReg
923              ? Register(getSubReg(ValueReg, getSubRegFromChannel(Lane)))
924              : ValueReg;
925       auto MIB = spillVGPRtoAGPR(ST, MI, Index, Lane, Sub, IsKill);
926       if (!MIB.getInstr())
927         break;
928       if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == 0)) {
929         MIB.addReg(ValueReg, RegState::ImplicitDefine);
930         NeedSuperRegDef = false;
931       }
932       if (IsSubReg || NeedSuperRegImpOperand) {
933         NeedSuperRegImpOperand = true;
934         unsigned State = SrcDstRegState;
935         if (Lane + 1 != LaneE)
936           State &= ~RegState::Kill;
937         MIB.addReg(ValueReg, RegState::Implicit | State);
938       }
939     }
940 
941     if (Lane == LaneE) // Fully spilled into AGPRs.
942       continue;
943 
944     // Offset in bytes from the beginning of the ValueReg to its portion we
945     // still need to spill. It may differ from RegOffset if a portion of
946     // current SubReg has been already spilled into AGPRs by the loop above.
947     unsigned RemRegOffset = Lane * 4;
948     unsigned RemEltSize = EltSize - (RemRegOffset - RegOffset);
949     if (RemEltSize != EltSize) { // Partially spilled to AGPRs
950       assert(IsFlat && EltSize > 4);
951 
952       unsigned NumRegs = RemEltSize / 4;
953       SubReg = Register(getSubReg(ValueReg,
954                         getSubRegFromChannel(RemRegOffset / 4, NumRegs)));
955       unsigned Opc = getFlatScratchSpillOpcode(TII, LoadStoreOp, RemEltSize);
956       Desc = &TII->get(Opc);
957     }
958 
959     unsigned FinalReg = SubReg;
960 
961     if (IsAGPR) {
962       assert(EltSize == 4);
963 
964       if (!TmpReg) {
965         assert(RS && "Needs to have RegScavenger to spill an AGPR!");
966         // FIXME: change to scavengeRegisterBackwards()
967         TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
968         RS->setRegUsed(TmpReg);
969       }
970       if (IsStore) {
971         auto AccRead = BuildMI(*MBB, MI, DL,
972                                TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg)
973           .addReg(SubReg, getKillRegState(IsKill));
974         if (NeedSuperRegDef)
975           AccRead.addReg(ValueReg, RegState::ImplicitDefine);
976         AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse);
977       }
978       SubReg = TmpReg;
979     }
980 
981     MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RemRegOffset);
982     MachineMemOperand *NewMMO =
983         MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize,
984                                  commonAlignment(Alignment, RemRegOffset));
985 
986     auto MIB = BuildMI(*MBB, MI, DL, *Desc)
987                   .addReg(SubReg,
988                           getDefRegState(!IsStore) | getKillRegState(IsKill));
989     if (!IsFlat)
990       MIB.addReg(FuncInfo->getScratchRSrcReg());
991 
992     if (SOffset == AMDGPU::NoRegister) {
993       if (!IsFlat)
994         MIB.addImm(0);
995     } else {
996       MIB.addReg(SOffset, SOffsetRegState);
997     }
998     MIB.addImm(Offset + RemRegOffset)
999         .addImm(0) // glc
1000         .addImm(0) // slc
1001         .addImm(0); // tfe for MUBUF or dlc for FLAT
1002     if (!IsFlat)
1003       MIB.addImm(0) // dlc
1004          .addImm(0); // swz
1005     MIB.addMemOperand(NewMMO);
1006 
1007     if (!IsAGPR && NeedSuperRegDef)
1008       MIB.addReg(ValueReg, RegState::ImplicitDefine);
1009 
1010     if (!IsStore && TmpReg != AMDGPU::NoRegister) {
1011       MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32),
1012                     FinalReg)
1013         .addReg(TmpReg, RegState::Kill);
1014       MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
1015     }
1016 
1017     if (NeedSuperRegImpOperand)
1018       MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState);
1019   }
1020 
1021   if (ScratchOffsetRegDelta != 0) {
1022     // Subtract the offset we added to the ScratchOffset register.
1023     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), SOffset)
1024         .addReg(SOffset)
1025         .addImm(ScratchOffsetRegDelta);
1026   }
1027 }
1028 
1029 // Generate a VMEM access which loads or stores the VGPR containing an SGPR
1030 // spill such that all the lanes set in VGPRLanes are loaded or stored.
1031 // This generates exec mask manipulation and will use SGPRs available in MI
1032 // or VGPR lanes in the VGPR to save and restore the exec mask.
1033 void SIRegisterInfo::buildSGPRSpillLoadStore(MachineBasicBlock::iterator MI,
1034                                              int Index, int Offset,
1035                                              unsigned EltSize, Register VGPR,
1036                                              int64_t VGPRLanes,
1037                                              RegScavenger *RS,
1038                                              bool IsLoad) const {
1039   MachineBasicBlock *MBB = MI->getParent();
1040   MachineFunction *MF = MBB->getParent();
1041   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1042   const SIInstrInfo *TII = ST.getInstrInfo();
1043 
1044   Register SuperReg = MI->getOperand(0).getReg();
1045   const TargetRegisterClass *RC = getPhysRegClass(SuperReg);
1046   ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize);
1047   unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
1048   unsigned FirstPart = Offset * 32;
1049   unsigned ExecLane = 0;
1050 
1051   bool IsKill = MI->getOperand(0).isKill();
1052   const DebugLoc &DL = MI->getDebugLoc();
1053 
1054   // Cannot handle load/store to EXEC
1055   assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI &&
1056          SuperReg != AMDGPU::EXEC && "exec should never spill");
1057 
1058   // On Wave32 only handle EXEC_LO.
1059   // On Wave64 only update EXEC_HI if there is sufficent space for a copy.
1060   bool OnlyExecLo = isWave32 || NumSubRegs == 1 || SuperReg == AMDGPU::EXEC_HI;
1061 
1062   unsigned ExecMovOpc = OnlyExecLo ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
1063   Register ExecReg = OnlyExecLo ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1064   Register SavedExecReg;
1065 
1066   // Backup EXEC
1067   if (OnlyExecLo) {
1068     SavedExecReg =
1069         NumSubRegs == 1
1070             ? SuperReg
1071             : Register(getSubReg(SuperReg, SplitParts[FirstPart + ExecLane]));
1072   } else {
1073     // If src/dst is an odd size it is possible subreg0 is not aligned.
1074     for (; ExecLane < (NumSubRegs - 1); ++ExecLane) {
1075       SavedExecReg = getMatchingSuperReg(
1076           getSubReg(SuperReg, SplitParts[FirstPart + ExecLane]), AMDGPU::sub0,
1077           &AMDGPU::SReg_64_XEXECRegClass);
1078       if (SavedExecReg)
1079         break;
1080     }
1081   }
1082   assert(SavedExecReg);
1083   BuildMI(*MBB, MI, DL, TII->get(ExecMovOpc), SavedExecReg).addReg(ExecReg);
1084 
1085   // Setup EXEC
1086   BuildMI(*MBB, MI, DL, TII->get(ExecMovOpc), ExecReg).addImm(VGPRLanes);
1087 
1088   // Load/store VGPR
1089   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1090   assert(FrameInfo.getStackID(Index) != TargetStackID::SGPRSpill);
1091 
1092   Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF)
1093                           ? getBaseRegister()
1094                           : getFrameRegister(*MF);
1095 
1096   Align Alignment = FrameInfo.getObjectAlign(Index);
1097   MachinePointerInfo PtrInfo =
1098       MachinePointerInfo::getFixedStack(*MF, Index);
1099   MachineMemOperand *MMO = MF->getMachineMemOperand(
1100       PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore,
1101       EltSize, Alignment);
1102 
1103   if (IsLoad) {
1104     unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
1105                                           : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
1106     buildSpillLoadStore(MI, Opc,
1107           Index,
1108           VGPR, false,
1109           FrameReg,
1110           Offset * EltSize, MMO,
1111           RS);
1112   } else {
1113     unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
1114                                           : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
1115     buildSpillLoadStore(MI, Opc, Index, VGPR,
1116                         IsKill, FrameReg,
1117                         Offset * EltSize, MMO, RS);
1118     // This only ever adds one VGPR spill
1119     MFI->addToSpilledVGPRs(1);
1120   }
1121 
1122   // Restore EXEC
1123   BuildMI(*MBB, MI, DL, TII->get(ExecMovOpc), ExecReg)
1124       .addReg(SavedExecReg, getKillRegState(IsLoad || IsKill));
1125 
1126   // Restore clobbered SGPRs
1127   if (IsLoad) {
1128     // Nothing to do; register will be overwritten
1129   } else if (!IsKill) {
1130     // Restore SGPRs from appropriate VGPR lanes
1131     if (!OnlyExecLo) {
1132       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
1133               getSubReg(SuperReg, SplitParts[FirstPart + ExecLane + 1]))
1134           .addReg(VGPR)
1135           .addImm(ExecLane + 1);
1136     }
1137     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
1138             NumSubRegs == 1 ? SavedExecReg
1139                             : Register(getSubReg(
1140                                   SuperReg, SplitParts[FirstPart + ExecLane])))
1141         .addReg(VGPR, RegState::Kill)
1142         .addImm(ExecLane);
1143   }
1144 }
1145 
1146 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
1147                                int Index,
1148                                RegScavenger *RS,
1149                                bool OnlyToVGPR) const {
1150   MachineBasicBlock *MBB = MI->getParent();
1151   MachineFunction *MF = MBB->getParent();
1152   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1153 
1154   ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
1155     = MFI->getSGPRToVGPRSpills(Index);
1156   bool SpillToVGPR = !VGPRSpills.empty();
1157   if (OnlyToVGPR && !SpillToVGPR)
1158     return false;
1159 
1160   const SIInstrInfo *TII = ST.getInstrInfo();
1161 
1162   Register SuperReg = MI->getOperand(0).getReg();
1163   bool IsKill = MI->getOperand(0).isKill();
1164   const DebugLoc &DL = MI->getDebugLoc();
1165 
1166   assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() &&
1167                          SuperReg != MFI->getFrameOffsetReg()));
1168 
1169   assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
1170   assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI &&
1171          SuperReg != AMDGPU::EXEC && "exec should never spill");
1172 
1173   unsigned EltSize = 4;
1174   const TargetRegisterClass *RC = getPhysRegClass(SuperReg);
1175 
1176   ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize);
1177   unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
1178 
1179   if (SpillToVGPR) {
1180     for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
1181       Register SubReg = NumSubRegs == 1
1182                             ? SuperReg
1183                             : Register(getSubReg(SuperReg, SplitParts[i]));
1184       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
1185 
1186       bool UseKill = IsKill && i == NumSubRegs - 1;
1187 
1188       // Mark the "old value of vgpr" input undef only if this is the first sgpr
1189       // spill to this specific vgpr in the first basic block.
1190       auto MIB =
1191           BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_WRITELANE_B32), Spill.VGPR)
1192               .addReg(SubReg, getKillRegState(UseKill))
1193               .addImm(Spill.Lane)
1194               .addReg(Spill.VGPR);
1195 
1196       if (i == 0 && NumSubRegs > 1) {
1197         // We may be spilling a super-register which is only partially defined,
1198         // and need to ensure later spills think the value is defined.
1199         MIB.addReg(SuperReg, RegState::ImplicitDefine);
1200       }
1201 
1202       if (NumSubRegs > 1)
1203         MIB.addReg(SuperReg, getKillRegState(UseKill) | RegState::Implicit);
1204 
1205       // FIXME: Since this spills to another register instead of an actual
1206       // frame index, we should delete the frame index when all references to
1207       // it are fixed.
1208     }
1209   } else {
1210     // Scavenged temporary VGPR to use. It must be scavenged once for any number
1211     // of spilled subregs.
1212     Register TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
1213     RS->setRegUsed(TmpVGPR);
1214 
1215     // SubReg carries the "Kill" flag when SubReg == SuperReg.
1216     unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill);
1217 
1218     unsigned PerVGPR = 32;
1219     unsigned NumVGPRs = (NumSubRegs + (PerVGPR - 1)) / PerVGPR;
1220     int64_t VGPRLanes = (1LL << std::min(PerVGPR, NumSubRegs)) - 1LL;
1221 
1222     for (unsigned Offset = 0; Offset < NumVGPRs; ++Offset) {
1223       unsigned TmpVGPRFlags = RegState::Undef;
1224 
1225       // Write sub registers into the VGPR
1226       for (unsigned i = Offset * PerVGPR,
1227                     e = std::min((Offset + 1) * PerVGPR, NumSubRegs);
1228            i < e; ++i) {
1229         Register SubReg = NumSubRegs == 1
1230                               ? SuperReg
1231                               : Register(getSubReg(SuperReg, SplitParts[i]));
1232 
1233         MachineInstrBuilder WriteLane =
1234             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_WRITELANE_B32), TmpVGPR)
1235                 .addReg(SubReg, SubKillState)
1236                 .addImm(i % PerVGPR)
1237                 .addReg(TmpVGPR, TmpVGPRFlags);
1238         TmpVGPRFlags = 0;
1239 
1240         // There could be undef components of a spilled super register.
1241         // TODO: Can we detect this and skip the spill?
1242         if (NumSubRegs > 1) {
1243           // The last implicit use of the SuperReg carries the "Kill" flag.
1244           unsigned SuperKillState = 0;
1245           if (i + 1 == NumSubRegs)
1246             SuperKillState |= getKillRegState(IsKill);
1247           WriteLane.addReg(SuperReg, RegState::Implicit | SuperKillState);
1248         }
1249       }
1250 
1251       // Write out VGPR
1252       buildSGPRSpillLoadStore(MI, Index, Offset, EltSize, TmpVGPR, VGPRLanes,
1253                               RS, false);
1254     }
1255   }
1256 
1257   MI->eraseFromParent();
1258   MFI->addToSpilledSGPRs(NumSubRegs);
1259   return true;
1260 }
1261 
1262 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
1263                                  int Index,
1264                                  RegScavenger *RS,
1265                                  bool OnlyToVGPR) const {
1266   MachineFunction *MF = MI->getParent()->getParent();
1267   MachineBasicBlock *MBB = MI->getParent();
1268   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1269 
1270   ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
1271     = MFI->getSGPRToVGPRSpills(Index);
1272   bool SpillToVGPR = !VGPRSpills.empty();
1273   if (OnlyToVGPR && !SpillToVGPR)
1274     return false;
1275 
1276   const SIInstrInfo *TII = ST.getInstrInfo();
1277   const DebugLoc &DL = MI->getDebugLoc();
1278 
1279   Register SuperReg = MI->getOperand(0).getReg();
1280 
1281   assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
1282   assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI &&
1283          SuperReg != AMDGPU::EXEC && "exec should never spill");
1284 
1285   unsigned EltSize = 4;
1286 
1287   const TargetRegisterClass *RC = getPhysRegClass(SuperReg);
1288 
1289   ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize);
1290   unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
1291 
1292   if (SpillToVGPR) {
1293     for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
1294       Register SubReg = NumSubRegs == 1
1295                             ? SuperReg
1296                             : Register(getSubReg(SuperReg, SplitParts[i]));
1297 
1298       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
1299       auto MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32), SubReg)
1300                      .addReg(Spill.VGPR)
1301                      .addImm(Spill.Lane);
1302       if (NumSubRegs > 1 && i == 0)
1303         MIB.addReg(SuperReg, RegState::ImplicitDefine);
1304     }
1305   } else {
1306     Register TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
1307     RS->setRegUsed(TmpVGPR);
1308 
1309     unsigned PerVGPR = 32;
1310     unsigned NumVGPRs = (NumSubRegs + (PerVGPR - 1)) / PerVGPR;
1311     int64_t VGPRLanes = (1LL << std::min(PerVGPR, NumSubRegs)) - 1LL;
1312 
1313     for (unsigned Offset = 0; Offset < NumVGPRs; ++Offset) {
1314       // Load in VGPR data
1315       buildSGPRSpillLoadStore(MI, Index, Offset, EltSize, TmpVGPR, VGPRLanes,
1316                               RS, true);
1317 
1318       // Unpack lanes
1319       for (unsigned i = Offset * PerVGPR,
1320                     e = std::min((Offset + 1) * PerVGPR, NumSubRegs);
1321            i < e; ++i) {
1322         Register SubReg = NumSubRegs == 1
1323                               ? SuperReg
1324                               : Register(getSubReg(SuperReg, SplitParts[i]));
1325 
1326         bool LastSubReg = (i + 1 == e);
1327         auto MIB =
1328             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32), SubReg)
1329                 .addReg(TmpVGPR, getKillRegState(LastSubReg))
1330                 .addImm(i);
1331         if (NumSubRegs > 1 && i == 0)
1332           MIB.addReg(SuperReg, RegState::ImplicitDefine);
1333       }
1334     }
1335   }
1336 
1337   MI->eraseFromParent();
1338   return true;
1339 }
1340 
1341 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to
1342 /// a VGPR and the stack slot can be safely eliminated when all other users are
1343 /// handled.
1344 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex(
1345   MachineBasicBlock::iterator MI,
1346   int FI,
1347   RegScavenger *RS) const {
1348   switch (MI->getOpcode()) {
1349   case AMDGPU::SI_SPILL_S1024_SAVE:
1350   case AMDGPU::SI_SPILL_S512_SAVE:
1351   case AMDGPU::SI_SPILL_S256_SAVE:
1352   case AMDGPU::SI_SPILL_S192_SAVE:
1353   case AMDGPU::SI_SPILL_S160_SAVE:
1354   case AMDGPU::SI_SPILL_S128_SAVE:
1355   case AMDGPU::SI_SPILL_S96_SAVE:
1356   case AMDGPU::SI_SPILL_S64_SAVE:
1357   case AMDGPU::SI_SPILL_S32_SAVE:
1358     return spillSGPR(MI, FI, RS, true);
1359   case AMDGPU::SI_SPILL_S1024_RESTORE:
1360   case AMDGPU::SI_SPILL_S512_RESTORE:
1361   case AMDGPU::SI_SPILL_S256_RESTORE:
1362   case AMDGPU::SI_SPILL_S192_RESTORE:
1363   case AMDGPU::SI_SPILL_S160_RESTORE:
1364   case AMDGPU::SI_SPILL_S128_RESTORE:
1365   case AMDGPU::SI_SPILL_S96_RESTORE:
1366   case AMDGPU::SI_SPILL_S64_RESTORE:
1367   case AMDGPU::SI_SPILL_S32_RESTORE:
1368     return restoreSGPR(MI, FI, RS, true);
1369   default:
1370     llvm_unreachable("not an SGPR spill instruction");
1371   }
1372 }
1373 
1374 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
1375                                         int SPAdj, unsigned FIOperandNum,
1376                                         RegScavenger *RS) const {
1377   MachineFunction *MF = MI->getParent()->getParent();
1378   MachineBasicBlock *MBB = MI->getParent();
1379   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1380   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1381   const SIInstrInfo *TII = ST.getInstrInfo();
1382   DebugLoc DL = MI->getDebugLoc();
1383 
1384   assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?");
1385 
1386   MachineOperand &FIOp = MI->getOperand(FIOperandNum);
1387   int Index = MI->getOperand(FIOperandNum).getIndex();
1388 
1389   Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF)
1390                           ? getBaseRegister()
1391                           : getFrameRegister(*MF);
1392 
1393   switch (MI->getOpcode()) {
1394     // SGPR register spill
1395     case AMDGPU::SI_SPILL_S1024_SAVE:
1396     case AMDGPU::SI_SPILL_S512_SAVE:
1397     case AMDGPU::SI_SPILL_S256_SAVE:
1398     case AMDGPU::SI_SPILL_S192_SAVE:
1399     case AMDGPU::SI_SPILL_S160_SAVE:
1400     case AMDGPU::SI_SPILL_S128_SAVE:
1401     case AMDGPU::SI_SPILL_S96_SAVE:
1402     case AMDGPU::SI_SPILL_S64_SAVE:
1403     case AMDGPU::SI_SPILL_S32_SAVE: {
1404       spillSGPR(MI, Index, RS);
1405       break;
1406     }
1407 
1408     // SGPR register restore
1409     case AMDGPU::SI_SPILL_S1024_RESTORE:
1410     case AMDGPU::SI_SPILL_S512_RESTORE:
1411     case AMDGPU::SI_SPILL_S256_RESTORE:
1412     case AMDGPU::SI_SPILL_S192_RESTORE:
1413     case AMDGPU::SI_SPILL_S160_RESTORE:
1414     case AMDGPU::SI_SPILL_S128_RESTORE:
1415     case AMDGPU::SI_SPILL_S96_RESTORE:
1416     case AMDGPU::SI_SPILL_S64_RESTORE:
1417     case AMDGPU::SI_SPILL_S32_RESTORE: {
1418       restoreSGPR(MI, Index, RS);
1419       break;
1420     }
1421 
1422     // VGPR register spill
1423     case AMDGPU::SI_SPILL_V1024_SAVE:
1424     case AMDGPU::SI_SPILL_V512_SAVE:
1425     case AMDGPU::SI_SPILL_V256_SAVE:
1426     case AMDGPU::SI_SPILL_V192_SAVE:
1427     case AMDGPU::SI_SPILL_V160_SAVE:
1428     case AMDGPU::SI_SPILL_V128_SAVE:
1429     case AMDGPU::SI_SPILL_V96_SAVE:
1430     case AMDGPU::SI_SPILL_V64_SAVE:
1431     case AMDGPU::SI_SPILL_V32_SAVE:
1432     case AMDGPU::SI_SPILL_A1024_SAVE:
1433     case AMDGPU::SI_SPILL_A512_SAVE:
1434     case AMDGPU::SI_SPILL_A256_SAVE:
1435     case AMDGPU::SI_SPILL_A192_SAVE:
1436     case AMDGPU::SI_SPILL_A160_SAVE:
1437     case AMDGPU::SI_SPILL_A128_SAVE:
1438     case AMDGPU::SI_SPILL_A96_SAVE:
1439     case AMDGPU::SI_SPILL_A64_SAVE:
1440     case AMDGPU::SI_SPILL_A32_SAVE: {
1441       const MachineOperand *VData = TII->getNamedOperand(*MI,
1442                                                          AMDGPU::OpName::vdata);
1443       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
1444              MFI->getStackPtrOffsetReg());
1445 
1446       unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
1447                                             : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
1448       buildSpillLoadStore(MI, Opc,
1449             Index,
1450             VData->getReg(), VData->isKill(),
1451             FrameReg,
1452             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
1453             *MI->memoperands_begin(),
1454             RS);
1455       MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode()));
1456       MI->eraseFromParent();
1457       break;
1458     }
1459     case AMDGPU::SI_SPILL_V32_RESTORE:
1460     case AMDGPU::SI_SPILL_V64_RESTORE:
1461     case AMDGPU::SI_SPILL_V96_RESTORE:
1462     case AMDGPU::SI_SPILL_V128_RESTORE:
1463     case AMDGPU::SI_SPILL_V160_RESTORE:
1464     case AMDGPU::SI_SPILL_V192_RESTORE:
1465     case AMDGPU::SI_SPILL_V256_RESTORE:
1466     case AMDGPU::SI_SPILL_V512_RESTORE:
1467     case AMDGPU::SI_SPILL_V1024_RESTORE:
1468     case AMDGPU::SI_SPILL_A32_RESTORE:
1469     case AMDGPU::SI_SPILL_A64_RESTORE:
1470     case AMDGPU::SI_SPILL_A96_RESTORE:
1471     case AMDGPU::SI_SPILL_A128_RESTORE:
1472     case AMDGPU::SI_SPILL_A160_RESTORE:
1473     case AMDGPU::SI_SPILL_A192_RESTORE:
1474     case AMDGPU::SI_SPILL_A256_RESTORE:
1475     case AMDGPU::SI_SPILL_A512_RESTORE:
1476     case AMDGPU::SI_SPILL_A1024_RESTORE: {
1477       const MachineOperand *VData = TII->getNamedOperand(*MI,
1478                                                          AMDGPU::OpName::vdata);
1479       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
1480              MFI->getStackPtrOffsetReg());
1481 
1482       unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
1483                                             : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
1484       buildSpillLoadStore(MI, Opc,
1485             Index,
1486             VData->getReg(), VData->isKill(),
1487             FrameReg,
1488             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
1489             *MI->memoperands_begin(),
1490             RS);
1491       MI->eraseFromParent();
1492       break;
1493     }
1494 
1495     default: {
1496       const DebugLoc &DL = MI->getDebugLoc();
1497 
1498       int64_t Offset = FrameInfo.getObjectOffset(Index);
1499       if (ST.enableFlatScratch()) {
1500         if (TII->isFLATScratch(*MI)) {
1501           // The offset is always swizzled, just replace it
1502           if (FrameReg)
1503             FIOp.ChangeToRegister(FrameReg, false);
1504 
1505           if (!Offset)
1506             return;
1507 
1508           MachineOperand *OffsetOp =
1509             TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1510           int64_t NewOffset = Offset + OffsetOp->getImm();
1511           if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS,
1512                                      true)) {
1513             OffsetOp->setImm(NewOffset);
1514             if (FrameReg)
1515               return;
1516             Offset = 0;
1517           }
1518 
1519           assert(!TII->getNamedOperand(*MI, AMDGPU::OpName::vaddr) &&
1520                  "Unexpected vaddr for flat scratch with a FI operand");
1521 
1522           // On GFX10 we have ST mode to use no registers for an address.
1523           // Otherwise we need to materialize 0 into an SGPR.
1524           if (!Offset && ST.hasFlatScratchSTMode()) {
1525             unsigned Opc = MI->getOpcode();
1526             unsigned NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc);
1527             MI->RemoveOperand(
1528                 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr));
1529             MI->setDesc(TII->get(NewOpc));
1530             return;
1531           }
1532         }
1533 
1534         if (!FrameReg) {
1535           FIOp.ChangeToImmediate(Offset);
1536           if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp))
1537             return;
1538         }
1539 
1540         // We need to use register here. Check if we can use an SGPR or need
1541         // a VGPR.
1542         FIOp.ChangeToRegister(AMDGPU::M0, false);
1543         bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, &FIOp);
1544 
1545         if (!Offset && FrameReg && UseSGPR) {
1546           FIOp.setReg(FrameReg);
1547           return;
1548         }
1549 
1550         const TargetRegisterClass *RC = UseSGPR ? &AMDGPU::SReg_32_XM0RegClass
1551                                                 : &AMDGPU::VGPR_32RegClass;
1552 
1553         Register TmpReg = RS->scavengeRegister(RC, MI, 0, !UseSGPR);
1554         FIOp.setReg(TmpReg);
1555         FIOp.setIsKill(true);
1556 
1557         if ((!FrameReg || !Offset) && TmpReg) {
1558           unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1559           auto MIB = BuildMI(*MBB, MI, DL, TII->get(Opc), TmpReg);
1560           if (FrameReg)
1561             MIB.addReg(FrameReg);
1562           else
1563             MIB.addImm(Offset);
1564 
1565           return;
1566         }
1567 
1568         Register TmpSReg =
1569             UseSGPR ? TmpReg
1570                     : RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0,
1571                                            !UseSGPR);
1572 
1573         // TODO: for flat scratch another attempt can be made with a VGPR index
1574         //       if no SGPRs can be scavenged.
1575         if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR))
1576           report_fatal_error("Cannot scavenge register in FI elimination!");
1577 
1578         if (!TmpSReg) {
1579           // Use frame register and restore it after.
1580           TmpSReg = FrameReg;
1581           FIOp.setReg(FrameReg);
1582           FIOp.setIsKill(false);
1583         }
1584 
1585         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), TmpSReg)
1586           .addReg(FrameReg)
1587           .addImm(Offset);
1588 
1589         if (!UseSGPR)
1590           BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
1591             .addReg(TmpSReg, RegState::Kill);
1592 
1593         if (TmpSReg == FrameReg) {
1594           // Undo frame register modification.
1595           BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_SUB_U32),
1596                   FrameReg)
1597             .addReg(FrameReg)
1598             .addImm(Offset);
1599         }
1600 
1601         return;
1602       }
1603 
1604       bool IsMUBUF = TII->isMUBUF(*MI);
1605 
1606       if (!IsMUBUF && !MFI->isEntryFunction()) {
1607         // Convert to a swizzled stack address by scaling by the wave size.
1608         //
1609         // In an entry function/kernel the offset is already swizzled.
1610 
1611         bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32;
1612         Register ResultReg =
1613             IsCopy ? MI->getOperand(0).getReg()
1614                    : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
1615 
1616         int64_t Offset = FrameInfo.getObjectOffset(Index);
1617         if (Offset == 0) {
1618           // XXX - This never happens because of emergency scavenging slot at 0?
1619           BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg)
1620             .addImm(ST.getWavefrontSizeLog2())
1621             .addReg(FrameReg);
1622         } else {
1623           if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) {
1624             // Reuse ResultReg in intermediate step.
1625             Register ScaledReg = ResultReg;
1626 
1627             BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64),
1628                     ScaledReg)
1629               .addImm(ST.getWavefrontSizeLog2())
1630               .addReg(FrameReg);
1631 
1632             const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32;
1633 
1634             // TODO: Fold if use instruction is another add of a constant.
1635             if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) {
1636               // FIXME: This can fail
1637               MIB.addImm(Offset);
1638               MIB.addReg(ScaledReg, RegState::Kill);
1639               if (!IsVOP2)
1640                 MIB.addImm(0); // clamp bit
1641             } else {
1642               assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 &&
1643                      "Need to reuse carry out register");
1644 
1645               // Use scavenged unused carry out as offset register.
1646               Register ConstOffsetReg;
1647               if (!isWave32)
1648                 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0);
1649               else
1650                 ConstOffsetReg = MIB.getReg(1);
1651 
1652               BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg)
1653                 .addImm(Offset);
1654               MIB.addReg(ConstOffsetReg, RegState::Kill);
1655               MIB.addReg(ScaledReg, RegState::Kill);
1656               MIB.addImm(0); // clamp bit
1657             }
1658           } else {
1659             // We have to produce a carry out, and there isn't a free SGPR pair
1660             // for it. We can keep the whole computation on the SALU to avoid
1661             // clobbering an additional register at the cost of an extra mov.
1662 
1663             // We may have 1 free scratch SGPR even though a carry out is
1664             // unavailable. Only one additional mov is needed.
1665             Register TmpScaledReg =
1666                 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false);
1667             Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg;
1668 
1669             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg)
1670               .addReg(FrameReg)
1671               .addImm(ST.getWavefrontSizeLog2());
1672             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), ScaledReg)
1673               .addReg(ScaledReg, RegState::Kill)
1674               .addImm(Offset);
1675             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg)
1676               .addReg(ScaledReg, RegState::Kill);
1677 
1678             // If there were truly no free SGPRs, we need to undo everything.
1679             if (!TmpScaledReg.isValid()) {
1680               BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg)
1681                 .addReg(ScaledReg, RegState::Kill)
1682                 .addImm(Offset);
1683               BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg)
1684                 .addReg(FrameReg)
1685                 .addImm(ST.getWavefrontSizeLog2());
1686             }
1687           }
1688         }
1689 
1690         // Don't introduce an extra copy if we're just materializing in a mov.
1691         if (IsCopy)
1692           MI->eraseFromParent();
1693         else
1694           FIOp.ChangeToRegister(ResultReg, false, false, true);
1695         return;
1696       }
1697 
1698       if (IsMUBUF) {
1699         // Disable offen so we don't need a 0 vgpr base.
1700         assert(static_cast<int>(FIOperandNum) ==
1701                AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1702                                           AMDGPU::OpName::vaddr));
1703 
1704         auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset);
1705         assert((SOffset.isReg() &&
1706                 SOffset.getReg() == MFI->getStackPtrOffsetReg()) ||
1707                (SOffset.isImm() && SOffset.getImm() == 0));
1708         if (SOffset.isReg()) {
1709           if (FrameReg == AMDGPU::NoRegister) {
1710             SOffset.ChangeToImmediate(0);
1711           } else {
1712             SOffset.setReg(FrameReg);
1713           }
1714         } else if (SOffset.isImm() && FrameReg != AMDGPU::NoRegister) {
1715           SOffset.ChangeToRegister(FrameReg, false);
1716         }
1717 
1718         int64_t Offset = FrameInfo.getObjectOffset(Index);
1719         int64_t OldImm
1720           = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm();
1721         int64_t NewOffset = OldImm + Offset;
1722 
1723         if (SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) &&
1724             buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) {
1725           MI->eraseFromParent();
1726           return;
1727         }
1728       }
1729 
1730       // If the offset is simply too big, don't convert to a scratch wave offset
1731       // relative index.
1732 
1733       FIOp.ChangeToImmediate(Offset);
1734       if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) {
1735         Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
1736         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
1737           .addImm(Offset);
1738         FIOp.ChangeToRegister(TmpReg, false, false, true);
1739       }
1740     }
1741   }
1742 }
1743 
1744 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const {
1745   return AMDGPUInstPrinter::getRegisterName(Reg);
1746 }
1747 
1748 const TargetRegisterClass *
1749 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) {
1750   if (BitWidth == 1)
1751     return &AMDGPU::VReg_1RegClass;
1752   if (BitWidth <= 16)
1753     return &AMDGPU::VGPR_LO16RegClass;
1754   if (BitWidth <= 32)
1755     return &AMDGPU::VGPR_32RegClass;
1756   if (BitWidth <= 64)
1757     return &AMDGPU::VReg_64RegClass;
1758   if (BitWidth <= 96)
1759     return &AMDGPU::VReg_96RegClass;
1760   if (BitWidth <= 128)
1761     return &AMDGPU::VReg_128RegClass;
1762   if (BitWidth <= 160)
1763     return &AMDGPU::VReg_160RegClass;
1764   if (BitWidth <= 192)
1765     return &AMDGPU::VReg_192RegClass;
1766   if (BitWidth <= 256)
1767     return &AMDGPU::VReg_256RegClass;
1768   if (BitWidth <= 512)
1769     return &AMDGPU::VReg_512RegClass;
1770   if (BitWidth <= 1024)
1771     return &AMDGPU::VReg_1024RegClass;
1772 
1773   return nullptr;
1774 }
1775 
1776 const TargetRegisterClass *
1777 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) {
1778   if (BitWidth <= 16)
1779     return &AMDGPU::AGPR_LO16RegClass;
1780   if (BitWidth <= 32)
1781     return &AMDGPU::AGPR_32RegClass;
1782   if (BitWidth <= 64)
1783     return &AMDGPU::AReg_64RegClass;
1784   if (BitWidth <= 96)
1785     return &AMDGPU::AReg_96RegClass;
1786   if (BitWidth <= 128)
1787     return &AMDGPU::AReg_128RegClass;
1788   if (BitWidth <= 160)
1789     return &AMDGPU::AReg_160RegClass;
1790   if (BitWidth <= 192)
1791     return &AMDGPU::AReg_192RegClass;
1792   if (BitWidth <= 256)
1793     return &AMDGPU::AReg_256RegClass;
1794   if (BitWidth <= 512)
1795     return &AMDGPU::AReg_512RegClass;
1796   if (BitWidth <= 1024)
1797     return &AMDGPU::AReg_1024RegClass;
1798 
1799   return nullptr;
1800 }
1801 
1802 const TargetRegisterClass *
1803 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) {
1804   if (BitWidth <= 16)
1805     return &AMDGPU::SGPR_LO16RegClass;
1806   if (BitWidth <= 32)
1807     return &AMDGPU::SReg_32RegClass;
1808   if (BitWidth <= 64)
1809     return &AMDGPU::SReg_64RegClass;
1810   if (BitWidth <= 96)
1811     return &AMDGPU::SGPR_96RegClass;
1812   if (BitWidth <= 128)
1813     return &AMDGPU::SGPR_128RegClass;
1814   if (BitWidth <= 160)
1815     return &AMDGPU::SGPR_160RegClass;
1816   if (BitWidth <= 192)
1817     return &AMDGPU::SGPR_192RegClass;
1818   if (BitWidth <= 256)
1819     return &AMDGPU::SGPR_256RegClass;
1820   if (BitWidth <= 512)
1821     return &AMDGPU::SGPR_512RegClass;
1822   if (BitWidth <= 1024)
1823     return &AMDGPU::SGPR_1024RegClass;
1824 
1825   return nullptr;
1826 }
1827 
1828 // FIXME: This is very slow. It might be worth creating a map from physreg to
1829 // register class.
1830 const TargetRegisterClass *
1831 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const {
1832   static const TargetRegisterClass *const BaseClasses[] = {
1833     &AMDGPU::VGPR_LO16RegClass,
1834     &AMDGPU::VGPR_HI16RegClass,
1835     &AMDGPU::SReg_LO16RegClass,
1836     &AMDGPU::AGPR_LO16RegClass,
1837     &AMDGPU::VGPR_32RegClass,
1838     &AMDGPU::SReg_32RegClass,
1839     &AMDGPU::AGPR_32RegClass,
1840     &AMDGPU::VReg_64RegClass,
1841     &AMDGPU::SReg_64RegClass,
1842     &AMDGPU::AReg_64RegClass,
1843     &AMDGPU::VReg_96RegClass,
1844     &AMDGPU::SReg_96RegClass,
1845     &AMDGPU::AReg_96RegClass,
1846     &AMDGPU::VReg_128RegClass,
1847     &AMDGPU::SReg_128RegClass,
1848     &AMDGPU::AReg_128RegClass,
1849     &AMDGPU::VReg_160RegClass,
1850     &AMDGPU::SReg_160RegClass,
1851     &AMDGPU::AReg_160RegClass,
1852     &AMDGPU::VReg_192RegClass,
1853     &AMDGPU::SReg_192RegClass,
1854     &AMDGPU::AReg_192RegClass,
1855     &AMDGPU::VReg_256RegClass,
1856     &AMDGPU::SReg_256RegClass,
1857     &AMDGPU::AReg_256RegClass,
1858     &AMDGPU::VReg_512RegClass,
1859     &AMDGPU::SReg_512RegClass,
1860     &AMDGPU::AReg_512RegClass,
1861     &AMDGPU::SReg_1024RegClass,
1862     &AMDGPU::VReg_1024RegClass,
1863     &AMDGPU::AReg_1024RegClass,
1864     &AMDGPU::SCC_CLASSRegClass,
1865     &AMDGPU::Pseudo_SReg_32RegClass,
1866     &AMDGPU::Pseudo_SReg_128RegClass,
1867   };
1868 
1869   for (const TargetRegisterClass *BaseClass : BaseClasses) {
1870     if (BaseClass->contains(Reg)) {
1871       return BaseClass;
1872     }
1873   }
1874   return nullptr;
1875 }
1876 
1877 // TODO: It might be helpful to have some target specific flags in
1878 // TargetRegisterClass to mark which classes are VGPRs to make this trivial.
1879 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const {
1880   unsigned Size = getRegSizeInBits(*RC);
1881   if (Size == 16) {
1882     return getCommonSubClass(&AMDGPU::VGPR_LO16RegClass, RC) != nullptr ||
1883            getCommonSubClass(&AMDGPU::VGPR_HI16RegClass, RC) != nullptr;
1884   }
1885   const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size);
1886   if (!VRC) {
1887     assert(Size < 32 && "Invalid register class size");
1888     return false;
1889   }
1890   return getCommonSubClass(VRC, RC) != nullptr;
1891 }
1892 
1893 bool SIRegisterInfo::hasAGPRs(const TargetRegisterClass *RC) const {
1894   unsigned Size = getRegSizeInBits(*RC);
1895   if (Size < 16)
1896     return false;
1897   const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size);
1898   if (!ARC) {
1899     assert(getVGPRClassForBitWidth(Size) && "Invalid register class size");
1900     return false;
1901   }
1902   return getCommonSubClass(ARC, RC) != nullptr;
1903 }
1904 
1905 const TargetRegisterClass *
1906 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const {
1907   unsigned Size = getRegSizeInBits(*SRC);
1908   const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size);
1909   assert(VRC && "Invalid register class size");
1910   return VRC;
1911 }
1912 
1913 const TargetRegisterClass *
1914 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const {
1915   unsigned Size = getRegSizeInBits(*SRC);
1916   const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size);
1917   assert(ARC && "Invalid register class size");
1918   return ARC;
1919 }
1920 
1921 const TargetRegisterClass *
1922 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const {
1923   unsigned Size = getRegSizeInBits(*VRC);
1924   if (Size == 32)
1925     return &AMDGPU::SGPR_32RegClass;
1926   const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size);
1927   assert(SRC && "Invalid register class size");
1928   return SRC;
1929 }
1930 
1931 const TargetRegisterClass *SIRegisterInfo::getSubRegClass(
1932                          const TargetRegisterClass *RC, unsigned SubIdx) const {
1933   if (SubIdx == AMDGPU::NoSubRegister)
1934     return RC;
1935 
1936   // We can assume that each lane corresponds to one 32-bit register.
1937   unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32;
1938   if (isSGPRClass(RC)) {
1939     if (Size == 32)
1940       RC = &AMDGPU::SGPR_32RegClass;
1941     else
1942       RC = getSGPRClassForBitWidth(Size);
1943   } else if (hasAGPRs(RC)) {
1944     RC = getAGPRClassForBitWidth(Size);
1945   } else {
1946     RC = getVGPRClassForBitWidth(Size);
1947   }
1948   assert(RC && "Invalid sub-register class size");
1949   return RC;
1950 }
1951 
1952 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const {
1953   if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&
1954       OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST)
1955     return !ST.hasMFMAInlineLiteralBug();
1956 
1957   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1958          OpType <= AMDGPU::OPERAND_SRC_LAST;
1959 }
1960 
1961 bool SIRegisterInfo::shouldRewriteCopySrc(
1962   const TargetRegisterClass *DefRC,
1963   unsigned DefSubReg,
1964   const TargetRegisterClass *SrcRC,
1965   unsigned SrcSubReg) const {
1966   // We want to prefer the smallest register class possible, so we don't want to
1967   // stop and rewrite on anything that looks like a subregister
1968   // extract. Operations mostly don't care about the super register class, so we
1969   // only want to stop on the most basic of copies between the same register
1970   // class.
1971   //
1972   // e.g. if we have something like
1973   // %0 = ...
1974   // %1 = ...
1975   // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2
1976   // %3 = COPY %2, sub0
1977   //
1978   // We want to look through the COPY to find:
1979   //  => %3 = COPY %0
1980 
1981   // Plain copy.
1982   return getCommonSubClass(DefRC, SrcRC) != nullptr;
1983 }
1984 
1985 /// Returns a lowest register that is not used at any point in the function.
1986 ///        If all registers are used, then this function will return
1987 ///         AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return
1988 ///         highest unused register.
1989 MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
1990                                               const TargetRegisterClass *RC,
1991                                               const MachineFunction &MF,
1992                                               bool ReserveHighestVGPR) const {
1993   if (ReserveHighestVGPR) {
1994     for (MCRegister Reg : reverse(*RC))
1995       if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
1996         return Reg;
1997   } else {
1998     for (MCRegister Reg : *RC)
1999       if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
2000         return Reg;
2001   }
2002   return MCRegister();
2003 }
2004 
2005 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC,
2006                                                    unsigned EltSize) const {
2007   const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC);
2008   assert(RegBitWidth >= 32 && RegBitWidth <= 1024);
2009 
2010   const unsigned RegDWORDs = RegBitWidth / 32;
2011   const unsigned EltDWORDs = EltSize / 4;
2012   assert(RegSplitParts.size() + 1 >= EltDWORDs);
2013 
2014   const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1];
2015   const unsigned NumParts = RegDWORDs / EltDWORDs;
2016 
2017   return makeArrayRef(Parts.data(), NumParts);
2018 }
2019 
2020 const TargetRegisterClass*
2021 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI,
2022                                   Register Reg) const {
2023   return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg);
2024 }
2025 
2026 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI,
2027                             Register Reg) const {
2028   const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
2029   // Registers without classes are unaddressable, SGPR-like registers.
2030   return RC && hasVGPRs(RC);
2031 }
2032 
2033 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI,
2034                             Register Reg) const {
2035   const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
2036 
2037   // Registers without classes are unaddressable, SGPR-like registers.
2038   return RC && hasAGPRs(RC);
2039 }
2040 
2041 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI,
2042                                     const TargetRegisterClass *SrcRC,
2043                                     unsigned SubReg,
2044                                     const TargetRegisterClass *DstRC,
2045                                     unsigned DstSubReg,
2046                                     const TargetRegisterClass *NewRC,
2047                                     LiveIntervals &LIS) const {
2048   unsigned SrcSize = getRegSizeInBits(*SrcRC);
2049   unsigned DstSize = getRegSizeInBits(*DstRC);
2050   unsigned NewSize = getRegSizeInBits(*NewRC);
2051 
2052   // Do not increase size of registers beyond dword, we would need to allocate
2053   // adjacent registers and constraint regalloc more than needed.
2054 
2055   // Always allow dword coalescing.
2056   if (SrcSize <= 32 || DstSize <= 32)
2057     return true;
2058 
2059   return NewSize <= DstSize || NewSize <= SrcSize;
2060 }
2061 
2062 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
2063                                              MachineFunction &MF) const {
2064   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2065 
2066   unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
2067                                                        MF.getFunction());
2068   switch (RC->getID()) {
2069   default:
2070     return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF);
2071   case AMDGPU::VGPR_32RegClassID:
2072   case AMDGPU::VGPR_LO16RegClassID:
2073   case AMDGPU::VGPR_HI16RegClassID:
2074     return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF));
2075   case AMDGPU::SGPR_32RegClassID:
2076   case AMDGPU::SGPR_LO16RegClassID:
2077     return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF));
2078   }
2079 }
2080 
2081 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF,
2082                                                 unsigned Idx) const {
2083   if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 ||
2084       Idx == AMDGPU::RegisterPressureSets::AGPR_32)
2085     return getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
2086                                const_cast<MachineFunction &>(MF));
2087 
2088   if (Idx == AMDGPU::RegisterPressureSets::SReg_32)
2089     return getRegPressureLimit(&AMDGPU::SGPR_32RegClass,
2090                                const_cast<MachineFunction &>(MF));
2091 
2092   llvm_unreachable("Unexpected register pressure set!");
2093 }
2094 
2095 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const {
2096   static const int Empty[] = { -1 };
2097 
2098   if (RegPressureIgnoredUnits[RegUnit])
2099     return Empty;
2100 
2101   return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit);
2102 }
2103 
2104 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
2105   // Not a callee saved register.
2106   return AMDGPU::SGPR30_SGPR31;
2107 }
2108 
2109 const TargetRegisterClass *
2110 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
2111                                          const RegisterBank &RB,
2112                                          const MachineRegisterInfo &MRI) const {
2113   switch (RB.getID()) {
2114   case AMDGPU::VGPRRegBankID:
2115     return getVGPRClassForBitWidth(std::max(32u, Size));
2116   case AMDGPU::VCCRegBankID:
2117     assert(Size == 1);
2118     return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass
2119                     : &AMDGPU::SReg_64_XEXECRegClass;
2120   case AMDGPU::SGPRRegBankID:
2121     return getSGPRClassForBitWidth(std::max(32u, Size));
2122   case AMDGPU::AGPRRegBankID:
2123     return getAGPRClassForBitWidth(std::max(32u, Size));
2124   default:
2125     llvm_unreachable("unknown register bank");
2126   }
2127 }
2128 
2129 const TargetRegisterClass *
2130 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
2131                                          const MachineRegisterInfo &MRI) const {
2132   const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg());
2133   if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>())
2134     return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI);
2135 
2136   const TargetRegisterClass *RC = RCOrRB.get<const TargetRegisterClass*>();
2137   return getAllocatableClass(RC);
2138 }
2139 
2140 MCRegister SIRegisterInfo::getVCC() const {
2141   return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC;
2142 }
2143 
2144 const TargetRegisterClass *
2145 SIRegisterInfo::getRegClass(unsigned RCID) const {
2146   switch ((int)RCID) {
2147   case AMDGPU::SReg_1RegClassID:
2148     return getBoolRC();
2149   case AMDGPU::SReg_1_XEXECRegClassID:
2150     return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass
2151       : &AMDGPU::SReg_64_XEXECRegClass;
2152   case -1:
2153     return nullptr;
2154   default:
2155     return AMDGPUGenRegisterInfo::getRegClass(RCID);
2156   }
2157 }
2158 
2159 // Find reaching register definition
2160 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg,
2161                                               MachineInstr &Use,
2162                                               MachineRegisterInfo &MRI,
2163                                               LiveIntervals *LIS) const {
2164   auto &MDT = LIS->getAnalysis<MachineDominatorTree>();
2165   SlotIndex UseIdx = LIS->getInstructionIndex(Use);
2166   SlotIndex DefIdx;
2167 
2168   if (Reg.isVirtual()) {
2169     if (!LIS->hasInterval(Reg))
2170       return nullptr;
2171     LiveInterval &LI = LIS->getInterval(Reg);
2172     LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg)
2173                                   : MRI.getMaxLaneMaskForVReg(Reg);
2174     VNInfo *V = nullptr;
2175     if (LI.hasSubRanges()) {
2176       for (auto &S : LI.subranges()) {
2177         if ((S.LaneMask & SubLanes) == SubLanes) {
2178           V = S.getVNInfoAt(UseIdx);
2179           break;
2180         }
2181       }
2182     } else {
2183       V = LI.getVNInfoAt(UseIdx);
2184     }
2185     if (!V)
2186       return nullptr;
2187     DefIdx = V->def;
2188   } else {
2189     // Find last def.
2190     for (MCRegUnitIterator Units(Reg.asMCReg(), this); Units.isValid();
2191          ++Units) {
2192       LiveRange &LR = LIS->getRegUnit(*Units);
2193       if (VNInfo *V = LR.getVNInfoAt(UseIdx)) {
2194         if (!DefIdx.isValid() ||
2195             MDT.dominates(LIS->getInstructionFromIndex(DefIdx),
2196                           LIS->getInstructionFromIndex(V->def)))
2197           DefIdx = V->def;
2198       } else {
2199         return nullptr;
2200       }
2201     }
2202   }
2203 
2204   MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx);
2205 
2206   if (!Def || !MDT.dominates(Def, &Use))
2207     return nullptr;
2208 
2209   assert(Def->modifiesRegister(Reg, this));
2210 
2211   return Def;
2212 }
2213 
2214 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const {
2215   assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32);
2216 
2217   for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass,
2218                                          AMDGPU::SReg_32RegClass,
2219                                          AMDGPU::AGPR_32RegClass } ) {
2220     if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC))
2221       return Super;
2222   }
2223   if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16,
2224                                             &AMDGPU::VGPR_32RegClass)) {
2225       return Super;
2226   }
2227 
2228   return AMDGPU::NoRegister;
2229 }
2230 
2231 bool SIRegisterInfo::isConstantPhysReg(MCRegister PhysReg) const {
2232   switch (PhysReg) {
2233   case AMDGPU::SGPR_NULL:
2234   case AMDGPU::SRC_SHARED_BASE:
2235   case AMDGPU::SRC_PRIVATE_BASE:
2236   case AMDGPU::SRC_SHARED_LIMIT:
2237   case AMDGPU::SRC_PRIVATE_LIMIT:
2238     return true;
2239   default:
2240     return false;
2241   }
2242 }
2243 
2244 ArrayRef<MCPhysReg>
2245 SIRegisterInfo::getAllSGPR128(const MachineFunction &MF) const {
2246   return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(),
2247                       ST.getMaxNumSGPRs(MF) / 4);
2248 }
2249 
2250 ArrayRef<MCPhysReg>
2251 SIRegisterInfo::getAllSGPR64(const MachineFunction &MF) const {
2252   return makeArrayRef(AMDGPU::SGPR_64RegClass.begin(),
2253                       ST.getMaxNumSGPRs(MF) / 2);
2254 }
2255 
2256 ArrayRef<MCPhysReg>
2257 SIRegisterInfo::getAllSGPR32(const MachineFunction &MF) const {
2258   return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF));
2259 }
2260