1 //===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the RegisterBankInfo class for
10 /// AMDGPU.
11 ///
12 /// \par
13 ///
14 /// AMDGPU has unique register bank constraints that require special high level
15 /// strategies to deal with. There are two main true physical register banks
16 /// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a
17 /// sort of pseudo-register bank needed to represent SGPRs used in a vector
18 /// boolean context. There is also the AGPR bank, which is a special purpose
19 /// physical register bank present on some subtargets.
20 ///
21 /// Copying from VGPR to SGPR is generally illegal, unless the value is known to
22 /// be uniform. It is generally not valid to legalize operands by inserting
23 /// copies as on other targets. Operations which require uniform, SGPR operands
24 /// generally require scalarization by repeatedly executing the instruction,
25 /// activating each set of lanes using a unique set of input values. This is
26 /// referred to as a waterfall loop.
27 ///
28 /// \par Booleans
29 ///
30 /// Booleans (s1 values) requires special consideration. A vector compare result
31 /// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
32 /// register. These are represented with the VCC bank. During selection, we need
33 /// to be able to unambiguously go back from a register class to a register
34 /// bank. To distinguish whether an SGPR should use the SGPR or VCC register
35 /// bank, we need to know the use context type. An SGPR s1 value always means a
36 /// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets
37 /// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38 /// a 32-bit virtual register. Taken together, this means we need to adjust the
39 /// type of boolean operations to be regbank legal. All SALU booleans need to be
40 /// widened to 32-bits, and all VALU booleans need to be s1 values.
41 ///
42 /// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
43 /// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc
44 /// bank. A non-boolean source (such as a truncate from a 1-bit load from
45 /// memory) will require a copy to the VCC bank which will require clearing the
46 /// high bits and inserting a compare.
47 ///
48 /// \par Constant bus restriction
49 ///
50 /// VALU instructions have a limitation known as the constant bus
51 /// restriction. Most VALU instructions can use SGPR operands, but may read at
52 /// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
53 /// instructions). This is one unique SGPR, so the same SGPR may be used for
54 /// multiple operands. From a register bank perspective, any combination of
55 /// operands should be legal as an SGPR, but this is contextually dependent on
56 /// the SGPR operands all being the same register. There is therefore optimal to
57 /// choose the SGPR with the most uses to minimize the number of copies.
58 ///
59 /// We avoid trying to solve this problem in RegBankSelect. Any VALU G_*
60 /// operation should have its source operands all mapped to VGPRs (except for
61 /// VCC), inserting copies from any SGPR operands. This the most trival legal
62 /// mapping. Anything beyond the simplest 1:1 instruction selection would be too
63 /// complicated to solve here. Every optimization pattern or instruction
64 /// selected to multiple outputs would have to enforce this rule, and there
65 /// would be additional complexity in tracking this rule for every G_*
66 /// operation. By forcing all inputs to VGPRs, it also simplifies the task of
67 /// picking the optimal operand combination from a post-isel optimization pass.
68 ///
69 //===----------------------------------------------------------------------===//
70 
71 #include "AMDGPURegisterBankInfo.h"
72 
73 #include "AMDGPUGlobalISelUtils.h"
74 #include "AMDGPUInstrInfo.h"
75 #include "AMDGPUSubtarget.h"
76 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
77 #include "SIMachineFunctionInfo.h"
78 #include "SIRegisterInfo.h"
79 #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
80 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
81 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
82 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
83 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
84 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
85 #include "llvm/CodeGen/TargetRegisterInfo.h"
86 #include "llvm/CodeGen/TargetSubtargetInfo.h"
87 #include "llvm/IR/Constants.h"
88 
89 #define GET_TARGET_REGBANK_IMPL
90 #include "AMDGPUGenRegisterBank.inc"
91 
92 // This file will be TableGen'ed at some point.
93 #include "AMDGPUGenRegisterBankInfo.def"
94 
95 using namespace llvm;
96 using namespace MIPatternMatch;
97 
98 namespace {
99 
100 // Observer to apply a register bank to new registers created by LegalizerHelper.
101 class ApplyRegBankMapping final : public GISelChangeObserver {
102 private:
103   const AMDGPURegisterBankInfo &RBI;
104   MachineRegisterInfo &MRI;
105   const RegisterBank *NewBank;
106   SmallVector<MachineInstr *, 4> NewInsts;
107 
108 public:
109   ApplyRegBankMapping(const AMDGPURegisterBankInfo &RBI_,
110                       MachineRegisterInfo &MRI_, const RegisterBank *RB)
111     : RBI(RBI_), MRI(MRI_), NewBank(RB) {}
112 
113   ~ApplyRegBankMapping() {
114     for (MachineInstr *MI : NewInsts)
115       applyBank(*MI);
116   }
117 
118   /// Set any registers that don't have a set register class or bank to SALU.
119   void applyBank(MachineInstr &MI) {
120     const unsigned Opc = MI.getOpcode();
121     if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT ||
122         Opc == AMDGPU::G_SEXT) {
123       // LegalizerHelper wants to use the basic legalization artifacts when
124       // widening etc. We don't handle selection with vcc in artifact sources,
125       // so we need to use a sslect instead to handle these properly.
126       Register DstReg = MI.getOperand(0).getReg();
127       Register SrcReg = MI.getOperand(1).getReg();
128       const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, *RBI.TRI);
129       if (SrcBank == &AMDGPU::VCCRegBank) {
130         const LLT S32 = LLT::scalar(32);
131         assert(MRI.getType(SrcReg) == LLT::scalar(1));
132         assert(MRI.getType(DstReg) == S32);
133         assert(NewBank == &AMDGPU::VGPRRegBank);
134 
135         // Replace the extension with a select, which really uses the boolean
136         // source.
137         MachineIRBuilder B(MI);
138         auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1);
139         auto False = B.buildConstant(S32, 0);
140         B.buildSelect(DstReg, SrcReg, True, False);
141         MRI.setRegBank(True.getReg(0), *NewBank);
142         MRI.setRegBank(False.getReg(0), *NewBank);
143         MI.eraseFromParent();
144       }
145 
146       assert(!MRI.getRegClassOrRegBank(DstReg));
147       MRI.setRegBank(DstReg, *NewBank);
148       return;
149     }
150 
151 #ifndef NDEBUG
152     if (Opc == AMDGPU::G_TRUNC) {
153       Register DstReg = MI.getOperand(0).getReg();
154       const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
155       assert(DstBank != &AMDGPU::VCCRegBank);
156     }
157 #endif
158 
159     for (MachineOperand &Op : MI.operands()) {
160       if (!Op.isReg())
161         continue;
162 
163       // We may see physical registers if building a real MI
164       Register Reg = Op.getReg();
165       if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg))
166         continue;
167 
168       const RegisterBank *RB = NewBank;
169       if (MRI.getType(Reg) == LLT::scalar(1)) {
170         assert(NewBank == &AMDGPU::VGPRRegBank &&
171                "s1 operands should only be used for vector bools");
172         assert((MI.getOpcode() != AMDGPU::G_TRUNC &&
173                 MI.getOpcode() != AMDGPU::G_ANYEXT) &&
174                "not expecting legalization artifacts here");
175         RB = &AMDGPU::VCCRegBank;
176       }
177 
178       MRI.setRegBank(Reg, *RB);
179     }
180   }
181 
182   void erasingInstr(MachineInstr &MI) override {}
183 
184   void createdInstr(MachineInstr &MI) override {
185     // At this point, the instruction was just inserted and has no operands.
186     NewInsts.push_back(&MI);
187   }
188 
189   void changingInstr(MachineInstr &MI) override {}
190   void changedInstr(MachineInstr &MI) override {}
191 };
192 
193 }
194 AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
195     : AMDGPUGenRegisterBankInfo(),
196       Subtarget(ST),
197       TRI(Subtarget.getRegisterInfo()),
198       TII(Subtarget.getInstrInfo()) {
199 
200   // HACK: Until this is fully tablegen'd.
201   static llvm::once_flag InitializeRegisterBankFlag;
202 
203   static auto InitializeRegisterBankOnce = [this]() {
204     assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank &&
205            &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank &&
206            &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank);
207     (void)this;
208   };
209 
210   llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
211 }
212 
213 static bool isVectorRegisterBank(const RegisterBank &Bank) {
214   unsigned BankID = Bank.getID();
215   return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
216 }
217 
218 unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
219                                           const RegisterBank &Src,
220                                           unsigned Size) const {
221   // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
222   if (Dst.getID() == AMDGPU::SGPRRegBankID &&
223       (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
224     return std::numeric_limits<unsigned>::max();
225   }
226 
227   // Bool values are tricky, because the meaning is based on context. The SCC
228   // and VCC banks are for the natural scalar and vector conditions produced by
229   // a compare.
230   //
231   // Legalization doesn't know about the necessary context, so an s1 use may
232   // have been a truncate from an arbitrary value, in which case a copy (lowered
233   // as a compare with 0) needs to be inserted.
234   if (Size == 1 &&
235       (Dst.getID() == AMDGPU::SGPRRegBankID) &&
236       (isVectorRegisterBank(Src) ||
237        Src.getID() == AMDGPU::SGPRRegBankID ||
238        Src.getID() == AMDGPU::VCCRegBankID))
239     return std::numeric_limits<unsigned>::max();
240 
241   // There is no direct copy between AGPRs.
242   if (Dst.getID() == AMDGPU::AGPRRegBankID &&
243       Src.getID() == AMDGPU::AGPRRegBankID)
244     return 4;
245 
246   return RegisterBankInfo::copyCost(Dst, Src, Size);
247 }
248 
249 unsigned AMDGPURegisterBankInfo::getBreakDownCost(
250   const ValueMapping &ValMapping,
251   const RegisterBank *CurBank) const {
252   // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to
253   // VGPR.
254   // FIXME: Is there a better way to do this?
255   if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
256     return 10; // This is expensive.
257 
258   assert(ValMapping.NumBreakDowns == 2 &&
259          ValMapping.BreakDown[0].Length == 32 &&
260          ValMapping.BreakDown[0].StartIdx == 0 &&
261          ValMapping.BreakDown[1].Length == 32 &&
262          ValMapping.BreakDown[1].StartIdx == 32 &&
263          ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
264 
265   // 32-bit extract of a 64-bit value is just access of a subregister, so free.
266   // TODO: Cost of 0 hits assert, though it's not clear it's what we really
267   // want.
268 
269   // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
270   // alignment restrictions, but this probably isn't important.
271   return 1;
272 }
273 
274 const RegisterBank &
275 AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
276                                                LLT Ty) const {
277   if (&RC == &AMDGPU::SReg_1RegClass)
278     return AMDGPU::VCCRegBank;
279 
280   // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a
281   // VCC-like use.
282   if (TRI->isSGPRClass(&RC)) {
283     // FIXME: This probably came from a copy from a physical register, which
284     // should be inferrrable from the copied to-type. We don't have many boolean
285     // physical register constraints so just assume a normal SGPR for now.
286     if (!Ty.isValid())
287       return AMDGPU::SGPRRegBank;
288 
289     return Ty == LLT::scalar(1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
290   }
291 
292   return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
293 }
294 
295 template <unsigned NumOps>
296 RegisterBankInfo::InstructionMappings
297 AMDGPURegisterBankInfo::addMappingFromTable(
298     const MachineInstr &MI, const MachineRegisterInfo &MRI,
299     const std::array<unsigned, NumOps> RegSrcOpIdx,
300     ArrayRef<OpRegBankEntry<NumOps>> Table) const {
301 
302   InstructionMappings AltMappings;
303 
304   SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands());
305 
306   unsigned Sizes[NumOps];
307   for (unsigned I = 0; I < NumOps; ++I) {
308     Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg();
309     Sizes[I] = getSizeInBits(Reg, MRI, *TRI);
310   }
311 
312   for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) {
313     unsigned SizeI = getSizeInBits(MI.getOperand(I).getReg(), MRI, *TRI);
314     Operands[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeI);
315   }
316 
317   // getInstrMapping's default mapping uses ID 1, so start at 2.
318   unsigned MappingID = 2;
319   for (const auto &Entry : Table) {
320     for (unsigned I = 0; I < NumOps; ++I) {
321       int OpIdx = RegSrcOpIdx[I];
322       Operands[OpIdx] = AMDGPU::getValueMapping(Entry.RegBanks[I], Sizes[I]);
323     }
324 
325     AltMappings.push_back(&getInstructionMapping(MappingID++, Entry.Cost,
326                                                  getOperandsMapping(Operands),
327                                                  Operands.size()));
328   }
329 
330   return AltMappings;
331 }
332 
333 RegisterBankInfo::InstructionMappings
334 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
335     const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
336   switch (MI.getIntrinsicID()) {
337   case Intrinsic::amdgcn_readlane: {
338     static const OpRegBankEntry<3> Table[2] = {
339       // Perfectly legal.
340       { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 },
341 
342       // Need a readfirstlane for the index.
343       { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
344     };
345 
346     const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
347     return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
348   }
349   case Intrinsic::amdgcn_writelane: {
350     static const OpRegBankEntry<4> Table[4] = {
351       // Perfectly legal.
352       { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 },
353 
354       // Need readfirstlane of first op
355       { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
356 
357       // Need readfirstlane of second op
358       { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
359 
360       // Need readfirstlane of both ops
361       { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 3 }
362     };
363 
364     // rsrc, voffset, offset
365     const std::array<unsigned, 4> RegSrcOpIdx = { { 0, 2, 3, 4 } };
366     return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
367   }
368   default:
369     return RegisterBankInfo::getInstrAlternativeMappings(MI);
370   }
371 }
372 
373 RegisterBankInfo::InstructionMappings
374 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
375     const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
376 
377   switch (MI.getIntrinsicID()) {
378   case Intrinsic::amdgcn_s_buffer_load: {
379     static const OpRegBankEntry<2> Table[4] = {
380       // Perfectly legal.
381       { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 },
382 
383       // Only need 1 register in loop
384       { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 300 },
385 
386       // Have to waterfall the resource.
387       { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1000 },
388 
389       // Have to waterfall the resource, and the offset.
390       { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1500 }
391     };
392 
393     // rsrc, offset
394     const std::array<unsigned, 2> RegSrcOpIdx = { { 2, 3 } };
395     return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
396   }
397   case Intrinsic::amdgcn_ds_ordered_add:
398   case Intrinsic::amdgcn_ds_ordered_swap: {
399     // VGPR = M0, VGPR
400     static const OpRegBankEntry<3> Table[2] = {
401       // Perfectly legal.
402       { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID  }, 1 },
403 
404       // Need a readfirstlane for m0
405       { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
406     };
407 
408     const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
409     return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
410   }
411   case Intrinsic::amdgcn_s_sendmsg:
412   case Intrinsic::amdgcn_s_sendmsghalt: {
413     // FIXME: Should have no register for immediate
414     static const OpRegBankEntry<1> Table[2] = {
415       // Perfectly legal.
416       { { AMDGPU::SGPRRegBankID }, 1 },
417 
418       // Need readlane
419       { { AMDGPU::VGPRRegBankID }, 3 }
420     };
421 
422     const std::array<unsigned, 1> RegSrcOpIdx = { { 2 } };
423     return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
424   }
425   default:
426     return RegisterBankInfo::getInstrAlternativeMappings(MI);
427   }
428 }
429 
430 static bool memOpHasNoClobbered(const MachineMemOperand *MMO) {
431   const Instruction *I = dyn_cast_or_null<Instruction>(MMO->getValue());
432   return I && I->getMetadata("amdgpu.noclobber");
433 }
434 
435 // FIXME: Returns uniform if there's no source value information. This is
436 // probably wrong.
437 static bool isScalarLoadLegal(const MachineInstr &MI) {
438   if (!MI.hasOneMemOperand())
439     return false;
440 
441   const MachineMemOperand *MMO = *MI.memoperands_begin();
442   const unsigned AS = MMO->getAddrSpace();
443   const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
444                        AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
445 
446   // There are no extending SMRD/SMEM loads, and they require 4-byte alignment.
447   return MMO->getSize() >= 4 && MMO->getAlign() >= Align(4) &&
448          // Can't do a scalar atomic load.
449          !MMO->isAtomic() &&
450          // Don't use scalar loads for volatile accesses to non-constant address
451          // spaces.
452          (IsConst || !MMO->isVolatile()) &&
453          // Memory must be known constant, or not written before this load.
454          (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) &&
455          AMDGPUInstrInfo::isUniformMMO(MMO);
456 }
457 
458 RegisterBankInfo::InstructionMappings
459 AMDGPURegisterBankInfo::getInstrAlternativeMappings(
460     const MachineInstr &MI) const {
461 
462   const MachineFunction &MF = *MI.getParent()->getParent();
463   const MachineRegisterInfo &MRI = MF.getRegInfo();
464 
465 
466   InstructionMappings AltMappings;
467   switch (MI.getOpcode()) {
468   case TargetOpcode::G_CONSTANT: {
469     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
470     if (Size == 1) {
471       static const OpRegBankEntry<1> Table[3] = {
472         { { AMDGPU::VGPRRegBankID }, 1 },
473         { { AMDGPU::SGPRRegBankID }, 1 },
474         { { AMDGPU::VCCRegBankID }, 1 }
475       };
476 
477       return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table);
478     }
479 
480     LLVM_FALLTHROUGH;
481   }
482   case TargetOpcode::G_FCONSTANT:
483   case TargetOpcode::G_FRAME_INDEX:
484   case TargetOpcode::G_GLOBAL_VALUE: {
485     static const OpRegBankEntry<1> Table[2] = {
486       { { AMDGPU::VGPRRegBankID }, 1 },
487       { { AMDGPU::SGPRRegBankID }, 1 }
488     };
489 
490     return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table);
491   }
492   case TargetOpcode::G_AND:
493   case TargetOpcode::G_OR:
494   case TargetOpcode::G_XOR: {
495     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
496 
497     if (Size == 1) {
498       // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
499       const InstructionMapping &SCCMapping = getInstructionMapping(
500         1, 1, getOperandsMapping(
501           {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32),
502            AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32),
503            AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32)}),
504         3); // Num Operands
505       AltMappings.push_back(&SCCMapping);
506 
507       const InstructionMapping &VCCMapping0 = getInstructionMapping(
508         2, 1, getOperandsMapping(
509           {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size),
510            AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size),
511            AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}),
512         3); // Num Operands
513       AltMappings.push_back(&VCCMapping0);
514       return AltMappings;
515     }
516 
517     if (Size != 64)
518       break;
519 
520     const InstructionMapping &SSMapping = getInstructionMapping(
521       1, 1, getOperandsMapping(
522         {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
523          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
524          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
525       3); // Num Operands
526     AltMappings.push_back(&SSMapping);
527 
528     const InstructionMapping &VVMapping = getInstructionMapping(
529       2, 2, getOperandsMapping(
530         {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
531          AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
532          AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
533       3); // Num Operands
534     AltMappings.push_back(&VVMapping);
535     break;
536   }
537   case TargetOpcode::G_LOAD:
538   case TargetOpcode::G_ZEXTLOAD:
539   case TargetOpcode::G_SEXTLOAD: {
540     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
541     LLT PtrTy = MRI.getType(MI.getOperand(1).getReg());
542     unsigned PtrSize = PtrTy.getSizeInBits();
543     unsigned AS = PtrTy.getAddressSpace();
544 
545     if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
546          AS != AMDGPUAS::PRIVATE_ADDRESS) &&
547         isScalarLoadLegal(MI)) {
548       const InstructionMapping &SSMapping = getInstructionMapping(
549           1, 1, getOperandsMapping(
550                     {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
551                      AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize)}),
552           2); // Num Operands
553       AltMappings.push_back(&SSMapping);
554     }
555 
556     const InstructionMapping &VVMapping = getInstructionMapping(
557         2, 1,
558         getOperandsMapping(
559             {AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
560              AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}),
561         2); // Num Operands
562     AltMappings.push_back(&VVMapping);
563 
564     // It may be possible to have a vgpr = load sgpr mapping here, because
565     // the mubuf instructions support this kind of load, but probably for only
566     // gfx7 and older.  However, the addressing mode matching in the instruction
567     // selector should be able to do a better job of detecting and selecting
568     // these kinds of loads from the vgpr = load vgpr mapping.
569 
570     return AltMappings;
571 
572   }
573   case TargetOpcode::G_SELECT: {
574     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
575     const InstructionMapping &SSMapping = getInstructionMapping(1, 1,
576       getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
577                           AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
578                           AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
579                           AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
580       4); // Num Operands
581     AltMappings.push_back(&SSMapping);
582 
583     const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
584       getOperandsMapping({AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
585                           AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
586                           AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
587                           AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
588       4); // Num Operands
589     AltMappings.push_back(&VVMapping);
590 
591     return AltMappings;
592   }
593   case TargetOpcode::G_SMIN:
594   case TargetOpcode::G_SMAX:
595   case TargetOpcode::G_UMIN:
596   case TargetOpcode::G_UMAX: {
597     static const OpRegBankEntry<3> Table[2] = {
598       { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 },
599 
600       // Scalar requires cmp+select, and extends if 16-bit.
601       // FIXME: Should there be separate costs for 32 and 16-bit
602       { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 3 }
603     };
604 
605     const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 1, 2 } };
606     return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
607   }
608   case TargetOpcode::G_UADDE:
609   case TargetOpcode::G_USUBE:
610   case TargetOpcode::G_SADDE:
611   case TargetOpcode::G_SSUBE: {
612     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
613     const InstructionMapping &SSMapping = getInstructionMapping(1, 1,
614       getOperandsMapping(
615         {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
616          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
617          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
618          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
619          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1)}),
620       5); // Num Operands
621     AltMappings.push_back(&SSMapping);
622 
623     const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
624       getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
625                           AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
626                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
627                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
628                           AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1)}),
629       5); // Num Operands
630     AltMappings.push_back(&VVMapping);
631     return AltMappings;
632   }
633   case AMDGPU::G_BRCOND: {
634     assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
635 
636     // TODO: Change type to 32 for scalar
637     const InstructionMapping &SMapping = getInstructionMapping(
638       1, 1, getOperandsMapping(
639         {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr}),
640       2); // Num Operands
641     AltMappings.push_back(&SMapping);
642 
643     const InstructionMapping &VMapping = getInstructionMapping(
644       1, 1, getOperandsMapping(
645         {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), nullptr }),
646       2); // Num Operands
647     AltMappings.push_back(&VMapping);
648     return AltMappings;
649   }
650   case AMDGPU::G_INTRINSIC:
651     return getInstrAlternativeMappingsIntrinsic(MI, MRI);
652   case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
653     return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI);
654   default:
655     break;
656   }
657   return RegisterBankInfo::getInstrAlternativeMappings(MI);
658 }
659 
660 void AMDGPURegisterBankInfo::split64BitValueForMapping(
661   MachineIRBuilder &B,
662   SmallVector<Register, 2> &Regs,
663   LLT HalfTy,
664   Register Reg) const {
665   assert(HalfTy.getSizeInBits() == 32);
666   MachineRegisterInfo *MRI = B.getMRI();
667   Register LoLHS = MRI->createGenericVirtualRegister(HalfTy);
668   Register HiLHS = MRI->createGenericVirtualRegister(HalfTy);
669   const RegisterBank *Bank = getRegBank(Reg, *MRI, *TRI);
670   MRI->setRegBank(LoLHS, *Bank);
671   MRI->setRegBank(HiLHS, *Bank);
672 
673   Regs.push_back(LoLHS);
674   Regs.push_back(HiLHS);
675 
676   B.buildInstr(AMDGPU::G_UNMERGE_VALUES)
677     .addDef(LoLHS)
678     .addDef(HiLHS)
679     .addUse(Reg);
680 }
681 
682 /// Replace the current type each register in \p Regs has with \p NewTy
683 static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs,
684                           LLT NewTy) {
685   for (Register Reg : Regs) {
686     assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits());
687     MRI.setType(Reg, NewTy);
688   }
689 }
690 
691 static LLT getHalfSizedType(LLT Ty) {
692   if (Ty.isVector()) {
693     assert(Ty.getNumElements() % 2 == 0);
694     return LLT::scalarOrVector(Ty.getNumElements() / 2, Ty.getElementType());
695   }
696 
697   assert(Ty.getSizeInBits() % 2 == 0);
698   return LLT::scalar(Ty.getSizeInBits() / 2);
699 }
700 
701 /// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If
702 /// any of the required SGPR operands are VGPRs, perform a waterfall loop to
703 /// execute the instruction for each unique combination of values in all lanes
704 /// in the wave. The block will be split such that rest of the instructions are
705 /// moved to a new block.
706 ///
707 /// Essentially performs this loop:
708 //
709 /// Save Execution Mask
710 /// For (Lane : Wavefront) {
711 ///   Enable Lane, Disable all other lanes
712 ///   SGPR = read SGPR value for current lane from VGPR
713 ///   VGPRResult[Lane] = use_op SGPR
714 /// }
715 /// Restore Execution Mask
716 ///
717 /// There is additional complexity to try for compare values to identify the
718 /// unique values used.
719 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
720   MachineIRBuilder &B,
721   iterator_range<MachineBasicBlock::iterator> Range,
722   SmallSet<Register, 4> &SGPROperandRegs,
723   MachineRegisterInfo &MRI) const {
724   SmallVector<Register, 4> ResultRegs;
725   SmallVector<Register, 4> InitResultRegs;
726   SmallVector<Register, 4> PhiRegs;
727 
728   // Track use registers which have already been expanded with a readfirstlane
729   // sequence. This may have multiple uses if moving a sequence.
730   DenseMap<Register, Register> WaterfalledRegMap;
731 
732   MachineBasicBlock &MBB = B.getMBB();
733   MachineFunction *MF = &B.getMF();
734 
735   const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
736   const unsigned WaveAndOpc = Subtarget.isWave32() ?
737     AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
738   const unsigned MovTermOpc = Subtarget.isWave32() ?
739     AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term;
740   const unsigned XorTermOpc = Subtarget.isWave32() ?
741     AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
742   const unsigned AndSaveExecOpc =  Subtarget.isWave32() ?
743     AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
744   const unsigned ExecReg =  Subtarget.isWave32() ?
745     AMDGPU::EXEC_LO : AMDGPU::EXEC;
746 
747 #ifndef NDEBUG
748   const int OrigRangeSize = std::distance(Range.begin(), Range.end());
749 #endif
750 
751   for (MachineInstr &MI : Range) {
752     for (MachineOperand &Def : MI.defs()) {
753       LLT ResTy = MRI.getType(Def.getReg());
754       const RegisterBank *DefBank = getRegBank(Def.getReg(), MRI, *TRI);
755       ResultRegs.push_back(Def.getReg());
756       Register InitReg = B.buildUndef(ResTy).getReg(0);
757       Register PhiReg = MRI.createGenericVirtualRegister(ResTy);
758       InitResultRegs.push_back(InitReg);
759       PhiRegs.push_back(PhiReg);
760       MRI.setRegBank(PhiReg, *DefBank);
761       MRI.setRegBank(InitReg, *DefBank);
762     }
763   }
764 
765   Register SaveExecReg = MRI.createVirtualRegister(WaveRC);
766   Register InitSaveExecReg = MRI.createVirtualRegister(WaveRC);
767 
768   // Don't bother using generic instructions/registers for the exec mask.
769   B.buildInstr(TargetOpcode::IMPLICIT_DEF)
770     .addDef(InitSaveExecReg);
771 
772   Register PhiExec = MRI.createVirtualRegister(WaveRC);
773   Register NewExec = MRI.createVirtualRegister(WaveRC);
774 
775   // To insert the loop we need to split the block. Move everything before this
776   // point to a new block, and insert a new empty block before this instruction.
777   MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
778   MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
779   MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
780   MachineFunction::iterator MBBI(MBB);
781   ++MBBI;
782   MF->insert(MBBI, LoopBB);
783   MF->insert(MBBI, RestoreExecBB);
784   MF->insert(MBBI, RemainderBB);
785 
786   LoopBB->addSuccessor(RestoreExecBB);
787   LoopBB->addSuccessor(LoopBB);
788 
789   // Move the rest of the block into a new block.
790   RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
791   RemainderBB->splice(RemainderBB->begin(), &MBB, Range.end(), MBB.end());
792 
793   MBB.addSuccessor(LoopBB);
794   RestoreExecBB->addSuccessor(RemainderBB);
795 
796   B.setInsertPt(*LoopBB, LoopBB->end());
797 
798   B.buildInstr(TargetOpcode::PHI)
799     .addDef(PhiExec)
800     .addReg(InitSaveExecReg)
801     .addMBB(&MBB)
802     .addReg(NewExec)
803     .addMBB(LoopBB);
804 
805   for (auto Result : zip(InitResultRegs, ResultRegs, PhiRegs)) {
806     B.buildInstr(TargetOpcode::G_PHI)
807       .addDef(std::get<2>(Result))
808       .addReg(std::get<0>(Result)) // Initial value / implicit_def
809       .addMBB(&MBB)
810       .addReg(std::get<1>(Result)) // Mid-loop value.
811       .addMBB(LoopBB);
812   }
813 
814   const DebugLoc &DL = B.getDL();
815 
816   MachineInstr &FirstInst = *Range.begin();
817 
818   // Move the instruction into the loop. Note we moved everything after
819   // Range.end() already into a new block, so Range.end() is no longer valid.
820   LoopBB->splice(LoopBB->end(), &MBB, Range.begin(), MBB.end());
821 
822   // Figure out the iterator range after splicing the instructions.
823   MachineBasicBlock::iterator NewBegin = FirstInst.getIterator();
824   auto NewEnd = LoopBB->end();
825 
826   MachineBasicBlock::iterator I = Range.begin();
827   B.setInsertPt(*LoopBB, I);
828 
829   Register CondReg;
830 
831   assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
832 
833   for (MachineInstr &MI : make_range(NewBegin, NewEnd)) {
834     for (MachineOperand &Op : MI.uses()) {
835       if (!Op.isReg() || Op.isDef())
836         continue;
837 
838       Register OldReg = Op.getReg();
839       if (!SGPROperandRegs.count(OldReg))
840         continue;
841 
842       // See if we already processed this register in another instruction in the
843       // sequence.
844       auto OldVal = WaterfalledRegMap.find(OldReg);
845       if (OldVal != WaterfalledRegMap.end()) {
846         Op.setReg(OldVal->second);
847         continue;
848       }
849 
850       Register OpReg = Op.getReg();
851       LLT OpTy = MRI.getType(OpReg);
852 
853       const RegisterBank *OpBank = getRegBank(OpReg, MRI, *TRI);
854       if (OpBank != &AMDGPU::VGPRRegBank) {
855         // Insert copy from AGPR to VGPR before the loop.
856         B.setMBB(MBB);
857         OpReg = B.buildCopy(OpTy, OpReg).getReg(0);
858         MRI.setRegBank(OpReg, AMDGPU::VGPRRegBank);
859         B.setInstr(*I);
860       }
861 
862       unsigned OpSize = OpTy.getSizeInBits();
863 
864       // Can only do a readlane of 32-bit pieces.
865       if (OpSize == 32) {
866         // Avoid extra copies in the simple case of one 32-bit register.
867         Register CurrentLaneOpReg
868           = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
869         MRI.setType(CurrentLaneOpReg, OpTy);
870 
871         constrainGenericRegister(OpReg, AMDGPU::VGPR_32RegClass, MRI);
872         // Read the next variant <- also loop target.
873         BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
874                 CurrentLaneOpReg)
875           .addReg(OpReg);
876 
877         Register NewCondReg = MRI.createVirtualRegister(WaveRC);
878         bool First = CondReg == AMDGPU::NoRegister;
879         if (First)
880           CondReg = NewCondReg;
881 
882         // Compare the just read M0 value to all possible Idx values.
883         B.buildInstr(AMDGPU::V_CMP_EQ_U32_e64)
884           .addDef(NewCondReg)
885           .addReg(CurrentLaneOpReg)
886           .addReg(OpReg);
887         Op.setReg(CurrentLaneOpReg);
888 
889         if (!First) {
890           Register AndReg = MRI.createVirtualRegister(WaveRC);
891 
892           // If there are multiple operands to consider, and the conditions.
893           B.buildInstr(WaveAndOpc)
894             .addDef(AndReg)
895             .addReg(NewCondReg)
896             .addReg(CondReg);
897           CondReg = AndReg;
898         }
899       } else {
900         LLT S32 = LLT::scalar(32);
901         SmallVector<Register, 8> ReadlanePieces;
902 
903         // The compares can be done as 64-bit, but the extract needs to be done
904         // in 32-bit pieces.
905 
906         bool Is64 = OpSize % 64 == 0;
907 
908         LLT UnmergeTy = OpSize % 64 == 0 ? LLT::scalar(64) : LLT::scalar(32);
909         unsigned CmpOp = OpSize % 64 == 0 ? AMDGPU::V_CMP_EQ_U64_e64
910           : AMDGPU::V_CMP_EQ_U32_e64;
911 
912         // The compares can be done as 64-bit, but the extract needs to be done
913         // in 32-bit pieces.
914 
915         // Insert the unmerge before the loop.
916 
917         B.setMBB(MBB);
918         auto Unmerge = B.buildUnmerge(UnmergeTy, OpReg);
919         B.setInstr(*I);
920 
921         unsigned NumPieces = Unmerge->getNumOperands() - 1;
922         for (unsigned PieceIdx = 0; PieceIdx != NumPieces; ++PieceIdx) {
923           Register UnmergePiece = Unmerge.getReg(PieceIdx);
924 
925           Register CurrentLaneOpReg;
926           if (Is64) {
927             Register CurrentLaneOpRegLo = MRI.createGenericVirtualRegister(S32);
928             Register CurrentLaneOpRegHi = MRI.createGenericVirtualRegister(S32);
929 
930             MRI.setRegClass(UnmergePiece, &AMDGPU::VReg_64RegClass);
931             MRI.setRegClass(CurrentLaneOpRegLo, &AMDGPU::SReg_32_XM0RegClass);
932             MRI.setRegClass(CurrentLaneOpRegHi, &AMDGPU::SReg_32_XM0RegClass);
933 
934             // Read the next variant <- also loop target.
935             BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
936                     CurrentLaneOpRegLo)
937               .addReg(UnmergePiece, 0, AMDGPU::sub0);
938 
939             // Read the next variant <- also loop target.
940             BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
941                     CurrentLaneOpRegHi)
942               .addReg(UnmergePiece, 0, AMDGPU::sub1);
943 
944             CurrentLaneOpReg =
945               B.buildMerge(LLT::scalar(64),
946                            {CurrentLaneOpRegLo, CurrentLaneOpRegHi})
947               .getReg(0);
948 
949             MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_64_XEXECRegClass);
950 
951             if (OpTy.getScalarSizeInBits() == 64) {
952               // If we need to produce a 64-bit element vector, so use the
953               // merged pieces
954               ReadlanePieces.push_back(CurrentLaneOpReg);
955             } else {
956               // 32-bit element type.
957               ReadlanePieces.push_back(CurrentLaneOpRegLo);
958               ReadlanePieces.push_back(CurrentLaneOpRegHi);
959             }
960           } else {
961             CurrentLaneOpReg = MRI.createGenericVirtualRegister(S32);
962             MRI.setRegClass(UnmergePiece, &AMDGPU::VGPR_32RegClass);
963             MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_32_XM0RegClass);
964 
965             // Read the next variant <- also loop target.
966             BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
967                     CurrentLaneOpReg)
968               .addReg(UnmergePiece);
969             ReadlanePieces.push_back(CurrentLaneOpReg);
970           }
971 
972           Register NewCondReg = MRI.createVirtualRegister(WaveRC);
973           bool First = CondReg == AMDGPU::NoRegister;
974           if (First)
975             CondReg = NewCondReg;
976 
977           B.buildInstr(CmpOp)
978             .addDef(NewCondReg)
979             .addReg(CurrentLaneOpReg)
980             .addReg(UnmergePiece);
981 
982           if (!First) {
983             Register AndReg = MRI.createVirtualRegister(WaveRC);
984 
985             // If there are multiple operands to consider, and the conditions.
986             B.buildInstr(WaveAndOpc)
987               .addDef(AndReg)
988               .addReg(NewCondReg)
989               .addReg(CondReg);
990             CondReg = AndReg;
991           }
992         }
993 
994         // FIXME: Build merge seems to switch to CONCAT_VECTORS but not
995         // BUILD_VECTOR
996         if (OpTy.isVector()) {
997           auto Merge = B.buildBuildVector(OpTy, ReadlanePieces);
998           Op.setReg(Merge.getReg(0));
999         } else {
1000           auto Merge = B.buildMerge(OpTy, ReadlanePieces);
1001           Op.setReg(Merge.getReg(0));
1002         }
1003 
1004         MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank);
1005       }
1006 
1007       // Make sure we don't re-process this register again.
1008       WaterfalledRegMap.insert(std::make_pair(OldReg, Op.getReg()));
1009     }
1010   }
1011 
1012   B.setInsertPt(*LoopBB, LoopBB->end());
1013 
1014   // Update EXEC, save the original EXEC value to VCC.
1015   B.buildInstr(AndSaveExecOpc)
1016     .addDef(NewExec)
1017     .addReg(CondReg, RegState::Kill);
1018 
1019   MRI.setSimpleHint(NewExec, CondReg);
1020 
1021   // Update EXEC, switch all done bits to 0 and all todo bits to 1.
1022   B.buildInstr(XorTermOpc)
1023     .addDef(ExecReg)
1024     .addReg(ExecReg)
1025     .addReg(NewExec);
1026 
1027   // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
1028   // s_cbranch_scc0?
1029 
1030   // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
1031   B.buildInstr(AMDGPU::S_CBRANCH_EXECNZ)
1032     .addMBB(LoopBB);
1033 
1034   // Save the EXEC mask before the loop.
1035   BuildMI(MBB, MBB.end(), DL, TII->get(MovTermOpc), SaveExecReg)
1036     .addReg(ExecReg);
1037 
1038   // Restore the EXEC mask after the loop.
1039   B.setMBB(*RestoreExecBB);
1040   B.buildInstr(MovTermOpc)
1041     .addDef(ExecReg)
1042     .addReg(SaveExecReg);
1043 
1044   // Set the insert point after the original instruction, so any new
1045   // instructions will be in the remainder.
1046   B.setInsertPt(*RemainderBB, RemainderBB->begin());
1047 
1048   return true;
1049 }
1050 
1051 // Return any unique registers used by \p MI at \p OpIndices that need to be
1052 // handled in a waterfall loop. Returns these registers in \p
1053 // SGPROperandRegs. Returns true if there are any operands to handle and a
1054 // waterfall loop is necessary.
1055 bool AMDGPURegisterBankInfo::collectWaterfallOperands(
1056   SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI,
1057   MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const {
1058   for (unsigned Op : OpIndices) {
1059     assert(MI.getOperand(Op).isUse());
1060     Register Reg = MI.getOperand(Op).getReg();
1061     const RegisterBank *OpBank = getRegBank(Reg, MRI, *TRI);
1062     if (OpBank->getID() != AMDGPU::SGPRRegBankID)
1063       SGPROperandRegs.insert(Reg);
1064   }
1065 
1066   // No operands need to be replaced, so no need to loop.
1067   return !SGPROperandRegs.empty();
1068 }
1069 
1070 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
1071   MachineIRBuilder &B, MachineInstr &MI, MachineRegisterInfo &MRI,
1072   ArrayRef<unsigned> OpIndices) const {
1073   // Use a set to avoid extra readfirstlanes in the case where multiple operands
1074   // are the same register.
1075   SmallSet<Register, 4> SGPROperandRegs;
1076 
1077   if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices))
1078     return false;
1079 
1080   MachineBasicBlock::iterator I = MI.getIterator();
1081   return executeInWaterfallLoop(B, make_range(I, std::next(I)),
1082                                 SGPROperandRegs, MRI);
1083 }
1084 
1085 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
1086   MachineInstr &MI, MachineRegisterInfo &MRI,
1087   ArrayRef<unsigned> OpIndices) const {
1088   MachineIRBuilder B(MI);
1089   return executeInWaterfallLoop(B, MI, MRI, OpIndices);
1090 }
1091 
1092 // Legalize an operand that must be an SGPR by inserting a readfirstlane.
1093 void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
1094     MachineInstr &MI, MachineRegisterInfo &MRI, unsigned OpIdx) const {
1095   Register Reg = MI.getOperand(OpIdx).getReg();
1096   const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
1097   if (Bank == &AMDGPU::SGPRRegBank)
1098     return;
1099 
1100   LLT Ty = MRI.getType(Reg);
1101   MachineIRBuilder B(MI);
1102 
1103   if (Bank != &AMDGPU::VGPRRegBank) {
1104     // We need to copy from AGPR to VGPR
1105     Reg = B.buildCopy(Ty, Reg).getReg(0);
1106     MRI.setRegBank(Reg, AMDGPU::VGPRRegBank);
1107   }
1108 
1109   Register SGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
1110   B.buildInstr(AMDGPU::V_READFIRSTLANE_B32)
1111     .addDef(SGPR)
1112     .addReg(Reg);
1113 
1114   MRI.setType(SGPR, Ty);
1115 
1116   const TargetRegisterClass *Constrained =
1117       constrainGenericRegister(Reg, AMDGPU::VGPR_32RegClass, MRI);
1118   (void)Constrained;
1119   assert(Constrained && "Failed to constrain readfirstlane src reg");
1120 
1121   MI.getOperand(OpIdx).setReg(SGPR);
1122 }
1123 
1124 /// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
1125 /// rest will be in the remainder.
1126 static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
1127   unsigned TotalSize = Ty.getSizeInBits();
1128   if (!Ty.isVector())
1129     return {LLT::scalar(FirstSize), LLT::scalar(TotalSize - FirstSize)};
1130 
1131   LLT EltTy = Ty.getElementType();
1132   unsigned EltSize = EltTy.getSizeInBits();
1133   assert(FirstSize % EltSize == 0);
1134 
1135   unsigned FirstPartNumElts = FirstSize / EltSize;
1136   unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1137 
1138   return {LLT::scalarOrVector(FirstPartNumElts, EltTy),
1139           LLT::scalarOrVector(RemainderElts, EltTy)};
1140 }
1141 
1142 static LLT widen96To128(LLT Ty) {
1143   if (!Ty.isVector())
1144     return LLT::scalar(128);
1145 
1146   LLT EltTy = Ty.getElementType();
1147   assert(128 % EltTy.getSizeInBits() == 0);
1148   return LLT::vector(128 / EltTy.getSizeInBits(), EltTy);
1149 }
1150 
1151 bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI,
1152                         const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1153                                               MachineRegisterInfo &MRI) const {
1154   Register DstReg = MI.getOperand(0).getReg();
1155   const LLT LoadTy = MRI.getType(DstReg);
1156   unsigned LoadSize = LoadTy.getSizeInBits();
1157   const unsigned MaxNonSmrdLoadSize = 128;
1158 
1159   const RegisterBank *PtrBank =
1160     OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1161   if (PtrBank == &AMDGPU::SGPRRegBank) {
1162     // If the pointer is an SGPR, we ordinarily have nothing to do.
1163     if (LoadSize != 96)
1164       return false;
1165 
1166     MachineMemOperand *MMO = *MI.memoperands_begin();
1167     Register PtrReg = MI.getOperand(1).getReg();
1168     // 96-bit loads are only available for vector loads. We need to split this
1169     // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1170 
1171     MachineIRBuilder B(MI);
1172     ApplyRegBankMapping O(*this, MRI, &AMDGPU::SGPRRegBank);
1173     GISelObserverWrapper Observer(&O);
1174     B.setChangeObserver(Observer);
1175 
1176     if (MMO->getAlign() < Align(16)) {
1177       LLT Part64, Part32;
1178       std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64);
1179       auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0);
1180       auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8);
1181 
1182       auto Undef = B.buildUndef(LoadTy);
1183       auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0);
1184       B.buildInsert(MI.getOperand(0), Ins0, Load1, 64);
1185     } else {
1186       LLT WiderTy = widen96To128(LoadTy);
1187       auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0);
1188       B.buildExtract(MI.getOperand(0), WideLoad, 0);
1189     }
1190 
1191     MI.eraseFromParent();
1192     return true;
1193   }
1194 
1195   // 128-bit loads are supported for all instruction types.
1196   if (LoadSize <= MaxNonSmrdLoadSize)
1197     return false;
1198 
1199   SmallVector<Register, 16> DefRegs(OpdMapper.getVRegs(0));
1200   SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(1));
1201 
1202   if (SrcRegs.empty())
1203     SrcRegs.push_back(MI.getOperand(1).getReg());
1204 
1205   assert(LoadSize % MaxNonSmrdLoadSize == 0);
1206 
1207   // RegBankSelect only emits scalar types, so we need to reset the pointer
1208   // operand to a pointer type.
1209   Register BasePtrReg = SrcRegs[0];
1210   LLT PtrTy = MRI.getType(MI.getOperand(1).getReg());
1211   MRI.setType(BasePtrReg, PtrTy);
1212 
1213   MachineIRBuilder B(MI);
1214 
1215   unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize;
1216   const LLT LoadSplitTy = LoadTy.divide(NumSplitParts);
1217   ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank);
1218   GISelObserverWrapper Observer(&O);
1219   B.setChangeObserver(Observer);
1220   LegalizerHelper Helper(B.getMF(), Observer, B);
1221 
1222   if (LoadTy.isVector()) {
1223     if (Helper.fewerElementsVector(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized)
1224       return false;
1225   } else {
1226     if (Helper.narrowScalar(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized)
1227       return false;
1228   }
1229 
1230   MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
1231   return true;
1232 }
1233 
1234 bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
1235   MachineInstr &MI,
1236   const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1237   MachineRegisterInfo &MRI) const {
1238   const MachineFunction &MF = *MI.getMF();
1239   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1240   const auto &TFI = *ST.getFrameLowering();
1241 
1242   // Guard in case the stack growth direction ever changes with scratch
1243   // instructions.
1244   if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown)
1245     return false;
1246 
1247   Register Dst = MI.getOperand(0).getReg();
1248   Register AllocSize = MI.getOperand(1).getReg();
1249   Align Alignment = assumeAligned(MI.getOperand(2).getImm());
1250 
1251   const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI);
1252 
1253   // TODO: Need to emit a wave reduction to get the maximum size.
1254   if (SizeBank != &AMDGPU::SGPRRegBank)
1255     return false;
1256 
1257   LLT PtrTy = MRI.getType(Dst);
1258   LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
1259 
1260   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1261   Register SPReg = Info->getStackPtrOffsetReg();
1262   ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank);
1263   GISelObserverWrapper Observer(&ApplyBank);
1264 
1265   MachineIRBuilder B(MI);
1266   B.setChangeObserver(Observer);
1267 
1268   auto WaveSize = B.buildConstant(LLT::scalar(32), ST.getWavefrontSizeLog2());
1269   auto ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize);
1270 
1271   auto SPCopy = B.buildCopy(PtrTy, SPReg);
1272   if (Alignment > TFI.getStackAlign()) {
1273     auto PtrAdd = B.buildPtrAdd(PtrTy, SPCopy, ScaledSize);
1274     B.buildMaskLowPtrBits(Dst, PtrAdd,
1275                           Log2(Alignment) + ST.getWavefrontSizeLog2());
1276   } else {
1277     B.buildPtrAdd(Dst, SPCopy, ScaledSize);
1278   }
1279 
1280   MI.eraseFromParent();
1281   return true;
1282 }
1283 
1284 bool AMDGPURegisterBankInfo::applyMappingImage(
1285     MachineInstr &MI, const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1286     MachineRegisterInfo &MRI, int RsrcIdx) const {
1287   const int NumDefs = MI.getNumExplicitDefs();
1288 
1289   // The reported argument index is relative to the IR intrinsic call arguments,
1290   // so we need to shift by the number of defs and the intrinsic ID.
1291   RsrcIdx += NumDefs + 1;
1292 
1293   // Insert copies to VGPR arguments.
1294   applyDefaultMapping(OpdMapper);
1295 
1296   // Fixup any SGPR arguments.
1297   SmallVector<unsigned, 4> SGPRIndexes;
1298   for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
1299     if (!MI.getOperand(I).isReg())
1300       continue;
1301 
1302     // If this intrinsic has a sampler, it immediately follows rsrc.
1303     if (I == RsrcIdx || I == RsrcIdx + 1)
1304       SGPRIndexes.push_back(I);
1305   }
1306 
1307   executeInWaterfallLoop(MI, MRI, SGPRIndexes);
1308   return true;
1309 }
1310 
1311 static Register getSrcRegIgnoringCopies(const MachineRegisterInfo &MRI,
1312                                         Register Reg) {
1313   MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1314   if (!Def)
1315     return Reg;
1316 
1317   // TODO: Guard against this being an implicit def
1318   return Def->getOperand(0).getReg();
1319 }
1320 
1321 // Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
1322 // the three offsets (voffset, soffset and instoffset)
1323 static unsigned setBufferOffsets(MachineIRBuilder &B,
1324                                  const AMDGPURegisterBankInfo &RBI,
1325                                  Register CombinedOffset, Register &VOffsetReg,
1326                                  Register &SOffsetReg, int64_t &InstOffsetVal,
1327                                  Align Alignment) {
1328   const LLT S32 = LLT::scalar(32);
1329   MachineRegisterInfo *MRI = B.getMRI();
1330 
1331   if (Optional<int64_t> Imm = getConstantVRegVal(CombinedOffset, *MRI)) {
1332     uint32_t SOffset, ImmOffset;
1333     if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset, &RBI.Subtarget,
1334                                  Alignment)) {
1335       VOffsetReg = B.buildConstant(S32, 0).getReg(0);
1336       SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
1337       InstOffsetVal = ImmOffset;
1338 
1339       B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1340       B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1341       return SOffset + ImmOffset;
1342     }
1343   }
1344 
1345   Register Base;
1346   unsigned Offset;
1347   MachineInstr *Unused;
1348 
1349   std::tie(Base, Offset, Unused)
1350     = AMDGPU::getBaseWithConstantOffset(*MRI, CombinedOffset);
1351 
1352   uint32_t SOffset, ImmOffset;
1353   if (Offset > 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset,
1354                                              &RBI.Subtarget, Alignment)) {
1355     if (RBI.getRegBank(Base, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) {
1356       VOffsetReg = Base;
1357       SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
1358       B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1359       InstOffsetVal = ImmOffset;
1360       return 0; // XXX - Why is this 0?
1361     }
1362 
1363     // If we have SGPR base, we can use it for soffset.
1364     if (SOffset == 0) {
1365       VOffsetReg = B.buildConstant(S32, 0).getReg(0);
1366       B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1367       SOffsetReg = Base;
1368       InstOffsetVal = ImmOffset;
1369       return 0; // XXX - Why is this 0?
1370     }
1371   }
1372 
1373   // Handle the variable sgpr + vgpr case.
1374   if (MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, *MRI)) {
1375     Register Src0 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(1).getReg());
1376     Register Src1 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(2).getReg());
1377 
1378     const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, *RBI.TRI);
1379     const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, *RBI.TRI);
1380 
1381     if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) {
1382       VOffsetReg = Src0;
1383       SOffsetReg = Src1;
1384       return 0;
1385     }
1386 
1387     if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) {
1388       VOffsetReg = Src1;
1389       SOffsetReg = Src0;
1390       return 0;
1391     }
1392   }
1393 
1394   // Ensure we have a VGPR for the combined offset. This could be an issue if we
1395   // have an SGPR offset and a VGPR resource.
1396   if (RBI.getRegBank(CombinedOffset, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) {
1397     VOffsetReg = CombinedOffset;
1398   } else {
1399     VOffsetReg = B.buildCopy(S32, CombinedOffset).getReg(0);
1400     B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1401   }
1402 
1403   SOffsetReg = B.buildConstant(S32, 0).getReg(0);
1404   B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1405   return 0;
1406 }
1407 
1408 bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
1409   const OperandsMapper &OpdMapper) const {
1410   MachineInstr &MI = OpdMapper.getMI();
1411   MachineRegisterInfo &MRI = OpdMapper.getMRI();
1412 
1413   const LLT S32 = LLT::scalar(32);
1414   Register Dst = MI.getOperand(0).getReg();
1415   LLT Ty = MRI.getType(Dst);
1416 
1417   const RegisterBank *RSrcBank =
1418     OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1419   const RegisterBank *OffsetBank =
1420     OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1421   if (RSrcBank == &AMDGPU::SGPRRegBank &&
1422       OffsetBank == &AMDGPU::SGPRRegBank)
1423     return true; // Legal mapping
1424 
1425   // FIXME: 96-bit case was widened during legalize. We neeed to narrow it back
1426   // here but don't have an MMO.
1427 
1428   unsigned LoadSize = Ty.getSizeInBits();
1429   int NumLoads = 1;
1430   if (LoadSize == 256 || LoadSize == 512) {
1431     NumLoads = LoadSize / 128;
1432     Ty = Ty.divide(NumLoads);
1433   }
1434 
1435   // Use the alignment to ensure that the required offsets will fit into the
1436   // immediate offsets.
1437   const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
1438 
1439   MachineIRBuilder B(MI);
1440   MachineFunction &MF = B.getMF();
1441 
1442   Register SOffset;
1443   Register VOffset;
1444   int64_t ImmOffset = 0;
1445 
1446   unsigned MMOOffset = setBufferOffsets(B, *this, MI.getOperand(2).getReg(),
1447                                         VOffset, SOffset, ImmOffset, Alignment);
1448 
1449   // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1450   // can, but we neeed to track an MMO for that.
1451   const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
1452   const Align MemAlign(4); // FIXME: ABI type alignment?
1453   MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
1454     MachinePointerInfo(),
1455     MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1456     MachineMemOperand::MOInvariant,
1457     MemSize, MemAlign);
1458   if (MMOOffset != 0)
1459     BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset, MemSize);
1460 
1461   // If only the offset is divergent, emit a MUBUF buffer load instead. We can
1462   // assume that the buffer is unswizzled.
1463 
1464   Register RSrc = MI.getOperand(1).getReg();
1465   Register VIndex = B.buildConstant(S32, 0).getReg(0);
1466   B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank);
1467 
1468   SmallVector<Register, 4> LoadParts(NumLoads);
1469 
1470   MachineBasicBlock::iterator MII = MI.getIterator();
1471   MachineInstrSpan Span(MII, &B.getMBB());
1472 
1473   for (int i = 0; i < NumLoads; ++i) {
1474     if (NumLoads == 1) {
1475       LoadParts[i] = Dst;
1476     } else {
1477       LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
1478       MRI.setRegBank(LoadParts[i], AMDGPU::VGPRRegBank);
1479     }
1480 
1481     MachineMemOperand *MMO = BaseMMO;
1482     if (i != 0)
1483       BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset + 16 * i, MemSize);
1484 
1485     B.buildInstr(AMDGPU::G_AMDGPU_BUFFER_LOAD)
1486       .addDef(LoadParts[i])       // vdata
1487       .addUse(RSrc)               // rsrc
1488       .addUse(VIndex)             // vindex
1489       .addUse(VOffset)            // voffset
1490       .addUse(SOffset)            // soffset
1491       .addImm(ImmOffset + 16 * i) // offset(imm)
1492       .addImm(0)                  // cachepolicy, swizzled buffer(imm)
1493       .addImm(0)                  // idxen(imm)
1494       .addMemOperand(MMO);
1495   }
1496 
1497   // TODO: If only the resource is a VGPR, it may be better to execute the
1498   // scalar load in the waterfall loop if the resource is expected to frequently
1499   // be dynamically uniform.
1500   if (RSrcBank != &AMDGPU::SGPRRegBank) {
1501     // Remove the original instruction to avoid potentially confusing the
1502     // waterfall loop logic.
1503     B.setInstr(*Span.begin());
1504     MI.eraseFromParent();
1505 
1506     SmallSet<Register, 4> OpsToWaterfall;
1507 
1508     OpsToWaterfall.insert(RSrc);
1509     executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
1510                            OpsToWaterfall, MRI);
1511   }
1512 
1513   if (NumLoads != 1) {
1514     if (Ty.isVector())
1515       B.buildConcatVectors(Dst, LoadParts);
1516     else
1517       B.buildMerge(Dst, LoadParts);
1518   }
1519 
1520   // We removed the instruction earlier with a waterfall loop.
1521   if (RSrcBank == &AMDGPU::SGPRRegBank)
1522     MI.eraseFromParent();
1523 
1524   return true;
1525 }
1526 
1527 bool AMDGPURegisterBankInfo::applyMappingBFEIntrinsic(
1528   const OperandsMapper &OpdMapper, bool Signed) const {
1529   MachineInstr &MI = OpdMapper.getMI();
1530   MachineRegisterInfo &MRI = OpdMapper.getMRI();
1531 
1532   // Insert basic copies
1533   applyDefaultMapping(OpdMapper);
1534 
1535   Register DstReg = MI.getOperand(0).getReg();
1536   LLT Ty = MRI.getType(DstReg);
1537 
1538   const LLT S32 = LLT::scalar(32);
1539 
1540   const RegisterBank *DstBank =
1541     OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
1542   if (DstBank == &AMDGPU::VGPRRegBank) {
1543     if (Ty == S32)
1544       return true;
1545 
1546     // TODO: 64-bit version is scalar only, so we need to expand this.
1547     return false;
1548   }
1549 
1550   Register SrcReg = MI.getOperand(2).getReg();
1551   Register OffsetReg = MI.getOperand(3).getReg();
1552   Register WidthReg = MI.getOperand(4).getReg();
1553 
1554   // The scalar form packs the offset and width in a single operand.
1555 
1556   ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank);
1557   GISelObserverWrapper Observer(&ApplyBank);
1558   MachineIRBuilder B(MI);
1559   B.setChangeObserver(Observer);
1560 
1561   // Ensure the high bits are clear to insert the offset.
1562   auto OffsetMask = B.buildConstant(S32, maskTrailingOnes<unsigned>(6));
1563   auto ClampOffset = B.buildAnd(S32, OffsetReg, OffsetMask);
1564 
1565   // Zeros out the low bits, so don't bother clamping the input value.
1566   auto ShiftWidth = B.buildShl(S32, WidthReg, B.buildConstant(S32, 16));
1567 
1568   // Transformation function, pack the offset and width of a BFE into
1569   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1570   // source, bits [5:0] contain the offset and bits [22:16] the width.
1571   auto MergedInputs = B.buildOr(S32, ClampOffset, ShiftWidth);
1572 
1573   // TODO: It might be worth using a pseudo here to avoid scc clobber and
1574   // register class constraints.
1575   unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) :
1576                              (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64);
1577 
1578   auto MIB = B.buildInstr(Opc, {DstReg}, {SrcReg, MergedInputs});
1579   if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
1580     llvm_unreachable("failed to constrain BFE");
1581 
1582   MI.eraseFromParent();
1583   return true;
1584 }
1585 
1586 // FIXME: Duplicated from LegalizerHelper
1587 static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
1588   switch (Opc) {
1589   case TargetOpcode::G_SMIN:
1590     return CmpInst::ICMP_SLT;
1591   case TargetOpcode::G_SMAX:
1592     return CmpInst::ICMP_SGT;
1593   case TargetOpcode::G_UMIN:
1594     return CmpInst::ICMP_ULT;
1595   case TargetOpcode::G_UMAX:
1596     return CmpInst::ICMP_UGT;
1597   default:
1598     llvm_unreachable("not in integer min/max");
1599   }
1600 }
1601 
1602 static unsigned minMaxToExtend(unsigned Opc) {
1603   switch (Opc) {
1604   case TargetOpcode::G_SMIN:
1605   case TargetOpcode::G_SMAX:
1606     return TargetOpcode::G_SEXT;
1607   case TargetOpcode::G_UMIN:
1608   case TargetOpcode::G_UMAX:
1609     return TargetOpcode::G_ZEXT;
1610   default:
1611     llvm_unreachable("not in integer min/max");
1612   }
1613 }
1614 
1615 // Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1616 // any illegal vector extend or unmerge operations.
1617 static std::pair<Register, Register>
1618 unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) {
1619   const LLT S32 = LLT::scalar(32);
1620   auto Bitcast = B.buildBitcast(S32, Src);
1621 
1622   if (ExtOpcode == TargetOpcode::G_SEXT) {
1623     auto ExtLo = B.buildSExtInReg(S32, Bitcast, 16);
1624     auto ShiftHi = B.buildAShr(S32, Bitcast, B.buildConstant(S32, 16));
1625     return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0));
1626   }
1627 
1628   auto ShiftHi = B.buildLShr(S32, Bitcast, B.buildConstant(S32, 16));
1629   if (ExtOpcode == TargetOpcode::G_ZEXT) {
1630     auto ExtLo = B.buildAnd(S32, Bitcast, B.buildConstant(S32, 0xffff));
1631     return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0));
1632   }
1633 
1634   assert(ExtOpcode == TargetOpcode::G_ANYEXT);
1635   return std::make_pair(Bitcast.getReg(0), ShiftHi.getReg(0));
1636 }
1637 
1638 static MachineInstr *buildExpandedScalarMinMax(MachineIRBuilder &B,
1639                                                CmpInst::Predicate Pred,
1640                                                Register Dst, Register Src0,
1641                                                Register Src1) {
1642   const LLT CmpType = LLT::scalar(32);
1643   auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1);
1644   return B.buildSelect(Dst, Cmp, Src0, Src1);
1645 }
1646 
1647 // FIXME: Duplicated from LegalizerHelper, except changing the boolean type.
1648 void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B,
1649                                                MachineInstr &MI) const {
1650   Register Dst = MI.getOperand(0).getReg();
1651   Register Src0 = MI.getOperand(1).getReg();
1652   Register Src1 = MI.getOperand(2).getReg();
1653 
1654   const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
1655   MachineInstr *Sel = buildExpandedScalarMinMax(B, Pred, Dst, Src0, Src1);
1656 
1657   Register CmpReg = Sel->getOperand(1).getReg();
1658   B.getMRI()->setRegBank(CmpReg, AMDGPU::SGPRRegBank);
1659   MI.eraseFromParent();
1660 }
1661 
1662 // For cases where only a single copy is inserted for matching register banks.
1663 // Replace the register in the instruction operand
1664 static bool substituteSimpleCopyRegs(
1665   const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) {
1666   SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx));
1667   if (!SrcReg.empty()) {
1668     assert(SrcReg.size() == 1);
1669     OpdMapper.getMI().getOperand(OpIdx).setReg(SrcReg[0]);
1670     return true;
1671   }
1672 
1673   return false;
1674 }
1675 
1676 /// Handle register layout difference for f16 images for some subtargets.
1677 Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
1678                                                 MachineRegisterInfo &MRI,
1679                                                 Register Reg) const {
1680   if (!Subtarget.hasUnpackedD16VMem())
1681     return Reg;
1682 
1683   const LLT S16 = LLT::scalar(16);
1684   LLT StoreVT = MRI.getType(Reg);
1685   if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
1686     return Reg;
1687 
1688   auto Unmerge = B.buildUnmerge(S16, Reg);
1689 
1690 
1691   SmallVector<Register, 4> WideRegs;
1692   for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1693     WideRegs.push_back(Unmerge.getReg(I));
1694 
1695   const LLT S32 = LLT::scalar(32);
1696   int NumElts = StoreVT.getNumElements();
1697 
1698   return B.buildMerge(LLT::vector(NumElts, S32), WideRegs).getReg(0);
1699 }
1700 
1701 static std::pair<Register, unsigned>
1702 getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
1703   int64_t Const;
1704   if (mi_match(Reg, MRI, m_ICst(Const)))
1705     return std::make_pair(Register(), Const);
1706 
1707   Register Base;
1708   if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const))))
1709     return std::make_pair(Base, Const);
1710 
1711   // TODO: Handle G_OR used for add case
1712   return std::make_pair(Reg, 0);
1713 }
1714 
1715 std::pair<Register, unsigned>
1716 AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
1717                                            Register OrigOffset) const {
1718   const unsigned MaxImm = 4095;
1719   Register BaseReg;
1720   unsigned ImmOffset;
1721   const LLT S32 = LLT::scalar(32);
1722 
1723   std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(),
1724                                                            OrigOffset);
1725 
1726   unsigned C1 = 0;
1727   if (ImmOffset != 0) {
1728     // If the immediate value is too big for the immoffset field, put the value
1729     // and -4096 into the immoffset field so that the value that is copied/added
1730     // for the voffset field is a multiple of 4096, and it stands more chance
1731     // of being CSEd with the copy/add for another similar load/store.
1732     // However, do not do that rounding down to a multiple of 4096 if that is a
1733     // negative number, as it appears to be illegal to have a negative offset
1734     // in the vgpr, even if adding the immediate offset makes it positive.
1735     unsigned Overflow = ImmOffset & ~MaxImm;
1736     ImmOffset -= Overflow;
1737     if ((int32_t)Overflow < 0) {
1738       Overflow += ImmOffset;
1739       ImmOffset = 0;
1740     }
1741 
1742     C1 = ImmOffset;
1743     if (Overflow != 0) {
1744       if (!BaseReg)
1745         BaseReg = B.buildConstant(S32, Overflow).getReg(0);
1746       else {
1747         auto OverflowVal = B.buildConstant(S32, Overflow);
1748         BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
1749       }
1750     }
1751   }
1752 
1753   if (!BaseReg)
1754     BaseReg = B.buildConstant(S32, 0).getReg(0);
1755 
1756   return {BaseReg, C1};
1757 }
1758 
1759 static bool isZero(Register Reg, MachineRegisterInfo &MRI) {
1760   int64_t C;
1761   return mi_match(Reg, MRI, m_ICst(C)) && C == 0;
1762 }
1763 
1764 static unsigned extractGLC(unsigned CachePolicy) {
1765   return CachePolicy & 1;
1766 }
1767 
1768 static unsigned extractSLC(unsigned CachePolicy) {
1769   return (CachePolicy >> 1) & 1;
1770 }
1771 
1772 static unsigned extractDLC(unsigned CachePolicy) {
1773   return (CachePolicy >> 2) & 1;
1774 }
1775 
1776 MachineInstr *
1777 AMDGPURegisterBankInfo::selectStoreIntrinsic(MachineIRBuilder &B,
1778                                              MachineInstr &MI) const {
1779    MachineRegisterInfo &MRI = *B.getMRI();
1780   executeInWaterfallLoop(B, MI, MRI, {2, 4});
1781 
1782   // FIXME: DAG lowering brokenly changes opcode based on FP vs. integer.
1783 
1784   Register VData = MI.getOperand(1).getReg();
1785   LLT Ty = MRI.getType(VData);
1786 
1787   int EltSize = Ty.getScalarSizeInBits();
1788   int Size = Ty.getSizeInBits();
1789 
1790   // FIXME: Broken integer truncstore.
1791   if (EltSize != 32)
1792     report_fatal_error("unhandled intrinsic store");
1793 
1794   // FIXME: Verifier should enforce 1 MMO for these intrinsics.
1795   const int MemSize = (*MI.memoperands_begin())->getSize();
1796 
1797 
1798   Register RSrc = MI.getOperand(2).getReg();
1799   Register VOffset = MI.getOperand(3).getReg();
1800   Register SOffset = MI.getOperand(4).getReg();
1801   unsigned CachePolicy = MI.getOperand(5).getImm();
1802 
1803   unsigned ImmOffset;
1804   std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
1805 
1806   const bool Offen = !isZero(VOffset, MRI);
1807 
1808   unsigned Opc = AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact;
1809   switch (8 * MemSize) {
1810   case 8:
1811     Opc = Offen ? AMDGPU::BUFFER_STORE_BYTE_OFFEN_exact :
1812                   AMDGPU::BUFFER_STORE_BYTE_OFFSET_exact;
1813     break;
1814   case 16:
1815     Opc = Offen ? AMDGPU::BUFFER_STORE_SHORT_OFFEN_exact :
1816                   AMDGPU::BUFFER_STORE_SHORT_OFFSET_exact;
1817     break;
1818   default:
1819     Opc = Offen ? AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact :
1820                   AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact;
1821     if (Size > 32)
1822       Opc = AMDGPU::getMUBUFOpcode(Opc, Size / 32);
1823     break;
1824   }
1825 
1826 
1827   // Set the insertion point back to the instruction in case it was moved into a
1828   // loop.
1829   B.setInstr(MI);
1830 
1831   MachineInstrBuilder MIB = B.buildInstr(Opc)
1832     .addUse(VData);
1833 
1834   if (Offen)
1835     MIB.addUse(VOffset);
1836 
1837   MIB.addUse(RSrc)
1838      .addUse(SOffset)
1839      .addImm(ImmOffset)
1840      .addImm(extractGLC(CachePolicy))
1841      .addImm(extractSLC(CachePolicy))
1842      .addImm(0) // tfe: FIXME: Remove from inst
1843      .addImm(extractDLC(CachePolicy))
1844      .cloneMemRefs(MI);
1845 
1846   // FIXME: We need a way to report failure from applyMappingImpl.
1847   // Insert constrain copies before inserting the loop.
1848   if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
1849     report_fatal_error("failed to constrain selected store intrinsic");
1850 
1851   return MIB;
1852 }
1853 
1854 bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
1855                                         Register SrcReg) const {
1856   MachineRegisterInfo &MRI = *B.getMRI();
1857   LLT SrcTy = MRI.getType(SrcReg);
1858   if (SrcTy.getSizeInBits() == 32) {
1859     // Use a v_mov_b32 here to make the exec dependency explicit.
1860     B.buildInstr(AMDGPU::V_MOV_B32_e32)
1861       .addDef(DstReg)
1862       .addUse(SrcReg);
1863     return constrainGenericRegister(DstReg, AMDGPU::VGPR_32RegClass, MRI) &&
1864            constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI);
1865   }
1866 
1867   Register TmpReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1868   Register TmpReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1869 
1870   B.buildInstr(AMDGPU::V_MOV_B32_e32)
1871     .addDef(TmpReg0)
1872     .addUse(SrcReg, 0, AMDGPU::sub0);
1873   B.buildInstr(AMDGPU::V_MOV_B32_e32)
1874     .addDef(TmpReg1)
1875     .addUse(SrcReg, 0, AMDGPU::sub1);
1876   B.buildInstr(AMDGPU::REG_SEQUENCE)
1877     .addDef(DstReg)
1878     .addUse(TmpReg0)
1879     .addImm(AMDGPU::sub0)
1880     .addUse(TmpReg1)
1881     .addImm(AMDGPU::sub1);
1882 
1883   return constrainGenericRegister(SrcReg, AMDGPU::SReg_64RegClass, MRI) &&
1884          constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI);
1885 }
1886 
1887 /// Utility function for pushing dynamic vector indexes with a constant offset
1888 /// into waterwall loops.
1889 static void reinsertVectorIndexAdd(MachineIRBuilder &B,
1890                                    MachineInstr &IdxUseInstr,
1891                                    unsigned OpIdx,
1892                                    unsigned ConstOffset) {
1893   MachineRegisterInfo &MRI = *B.getMRI();
1894   const LLT S32 = LLT::scalar(32);
1895   Register WaterfallIdx = IdxUseInstr.getOperand(OpIdx).getReg();
1896   B.setInsertPt(*IdxUseInstr.getParent(), IdxUseInstr.getIterator());
1897 
1898   auto MaterializedOffset = B.buildConstant(S32, ConstOffset);
1899 
1900   auto Add = B.buildAdd(S32, WaterfallIdx, MaterializedOffset);
1901   MRI.setRegBank(MaterializedOffset.getReg(0), AMDGPU::SGPRRegBank);
1902   MRI.setRegBank(Add.getReg(0), AMDGPU::SGPRRegBank);
1903   IdxUseInstr.getOperand(OpIdx).setReg(Add.getReg(0));
1904 }
1905 
1906 /// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1907 /// original 32-bit source value (to be inserted in the low part of the combined
1908 /// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1909 /// value.
1910 static void extendLow32IntoHigh32(MachineIRBuilder &B,
1911                                   Register Hi32Reg, Register Lo32Reg,
1912                                   unsigned ExtOpc,
1913                                   const RegisterBank &RegBank,
1914                                   bool IsBooleanSrc = false) {
1915   if (ExtOpc == AMDGPU::G_ZEXT) {
1916     B.buildConstant(Hi32Reg, 0);
1917   } else if (ExtOpc == AMDGPU::G_SEXT) {
1918     if (IsBooleanSrc) {
1919       // If we know the original source was an s1, the high half is the same as
1920       // the low.
1921       B.buildCopy(Hi32Reg, Lo32Reg);
1922     } else {
1923       // Replicate sign bit from 32-bit extended part.
1924       auto ShiftAmt = B.buildConstant(LLT::scalar(32), 31);
1925       B.getMRI()->setRegBank(ShiftAmt.getReg(0), RegBank);
1926       B.buildAShr(Hi32Reg, Lo32Reg, ShiftAmt);
1927     }
1928   } else {
1929     assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension");
1930     B.buildUndef(Hi32Reg);
1931   }
1932 }
1933 
1934 bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect(
1935   MachineInstr &MI, MachineRegisterInfo &MRI,
1936   const OperandsMapper &OpdMapper) const {
1937 
1938   Register VecReg = MI.getOperand(1).getReg();
1939   Register Idx = MI.getOperand(2).getReg();
1940 
1941   const RegisterBank &IdxBank =
1942     *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1943 
1944   bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
1945 
1946   LLT VecTy = MRI.getType(VecReg);
1947   unsigned EltSize = VecTy.getScalarSizeInBits();
1948   unsigned NumElem = VecTy.getNumElements();
1949 
1950   if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1951                                                   IsDivergentIdx))
1952     return false;
1953 
1954   MachineIRBuilder B(MI);
1955   LLT S32 = LLT::scalar(32);
1956 
1957   const RegisterBank &DstBank =
1958     *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
1959   const RegisterBank &SrcBank =
1960     *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1961 
1962   const RegisterBank &CCBank =
1963     (DstBank == AMDGPU::SGPRRegBank &&
1964      SrcBank == AMDGPU::SGPRRegBank &&
1965      IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
1966                                      : AMDGPU::VCCRegBank;
1967   LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1);
1968 
1969   if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
1970     Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
1971     MRI.setRegBank(Idx, AMDGPU::VGPRRegBank);
1972   }
1973 
1974   LLT EltTy = VecTy.getScalarType();
1975   SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
1976   unsigned NumLanes = DstRegs.size();
1977   if (!NumLanes)
1978     NumLanes = 1;
1979   else
1980     EltTy = MRI.getType(DstRegs[0]);
1981 
1982   auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg);
1983   SmallVector<Register, 2> Res(NumLanes);
1984   for (unsigned L = 0; L < NumLanes; ++L)
1985     Res[L] = UnmergeToEltTy.getReg(L);
1986 
1987   for (unsigned I = 1; I < NumElem; ++I) {
1988     auto IC = B.buildConstant(S32, I);
1989     MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
1990     auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC);
1991     MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
1992 
1993     for (unsigned L = 0; L < NumLanes; ++L) {
1994       auto S = B.buildSelect(EltTy, Cmp,
1995                              UnmergeToEltTy.getReg(I * NumLanes + L), Res[L]);
1996 
1997       for (unsigned N : { 0, 2, 3 })
1998         MRI.setRegBank(S->getOperand(N).getReg(), DstBank);
1999 
2000       Res[L] = S->getOperand(0).getReg();
2001     }
2002   }
2003 
2004   for (unsigned L = 0; L < NumLanes; ++L) {
2005     Register DstReg = (NumLanes == 1) ? MI.getOperand(0).getReg() : DstRegs[L];
2006     B.buildCopy(DstReg, Res[L]);
2007     MRI.setRegBank(DstReg, DstBank);
2008   }
2009 
2010   MRI.setRegBank(MI.getOperand(0).getReg(), DstBank);
2011   MI.eraseFromParent();
2012 
2013   return true;
2014 }
2015 
2016 bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect(
2017   MachineInstr &MI, MachineRegisterInfo &MRI,
2018   const OperandsMapper &OpdMapper) const {
2019 
2020   Register VecReg = MI.getOperand(1).getReg();
2021   Register Idx = MI.getOperand(3).getReg();
2022 
2023   const RegisterBank &IdxBank =
2024     *OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank;
2025 
2026   bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
2027 
2028   LLT VecTy = MRI.getType(VecReg);
2029   unsigned EltSize = VecTy.getScalarSizeInBits();
2030   unsigned NumElem = VecTy.getNumElements();
2031 
2032   if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
2033                                                   IsDivergentIdx))
2034     return false;
2035 
2036   MachineIRBuilder B(MI);
2037   LLT S32 = LLT::scalar(32);
2038 
2039   const RegisterBank &DstBank =
2040     *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2041   const RegisterBank &SrcBank =
2042     *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2043   const RegisterBank &InsBank =
2044     *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2045 
2046   const RegisterBank &CCBank =
2047     (DstBank == AMDGPU::SGPRRegBank &&
2048      SrcBank == AMDGPU::SGPRRegBank &&
2049      InsBank == AMDGPU::SGPRRegBank &&
2050      IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
2051                                      : AMDGPU::VCCRegBank;
2052   LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1);
2053 
2054   if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
2055     Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
2056     MRI.setRegBank(Idx, AMDGPU::VGPRRegBank);
2057   }
2058 
2059   LLT EltTy = VecTy.getScalarType();
2060   SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2061   unsigned NumLanes = InsRegs.size();
2062   if (!NumLanes) {
2063     NumLanes = 1;
2064     InsRegs.push_back(MI.getOperand(2).getReg());
2065   } else {
2066     EltTy = MRI.getType(InsRegs[0]);
2067   }
2068 
2069   auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg);
2070   SmallVector<Register, 16> Ops(NumElem * NumLanes);
2071 
2072   for (unsigned I = 0; I < NumElem; ++I) {
2073     auto IC = B.buildConstant(S32, I);
2074     MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
2075     auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC);
2076     MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
2077 
2078     for (unsigned L = 0; L < NumLanes; ++L) {
2079       auto S = B.buildSelect(EltTy, Cmp, InsRegs[L],
2080                              UnmergeToEltTy.getReg(I * NumLanes + L));
2081 
2082       for (unsigned N : { 0, 2, 3 })
2083         MRI.setRegBank(S->getOperand(N).getReg(), DstBank);
2084 
2085       Ops[I * NumLanes + L] = S->getOperand(0).getReg();
2086     }
2087   }
2088 
2089   LLT MergeTy = LLT::vector(Ops.size(), EltTy);
2090   if (MergeTy == MRI.getType(MI.getOperand(0).getReg())) {
2091     B.buildBuildVector(MI.getOperand(0), Ops);
2092   } else {
2093     auto Vec = B.buildBuildVector(MergeTy, Ops);
2094     MRI.setRegBank(Vec->getOperand(0).getReg(), DstBank);
2095     B.buildBitcast(MI.getOperand(0).getReg(), Vec);
2096   }
2097 
2098   MRI.setRegBank(MI.getOperand(0).getReg(), DstBank);
2099   MI.eraseFromParent();
2100 
2101   return true;
2102 }
2103 
2104 void AMDGPURegisterBankInfo::applyMappingImpl(
2105     const OperandsMapper &OpdMapper) const {
2106   MachineInstr &MI = OpdMapper.getMI();
2107   unsigned Opc = MI.getOpcode();
2108   MachineRegisterInfo &MRI = OpdMapper.getMRI();
2109   switch (Opc) {
2110   case AMDGPU::G_PHI: {
2111     Register DstReg = MI.getOperand(0).getReg();
2112     LLT DstTy = MRI.getType(DstReg);
2113     if (DstTy != LLT::scalar(1))
2114       break;
2115 
2116     const LLT S32 = LLT::scalar(32);
2117     const RegisterBank *DstBank =
2118       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2119     if (DstBank == &AMDGPU::VCCRegBank) {
2120       applyDefaultMapping(OpdMapper);
2121       // The standard handling only considers the result register bank for
2122       // phis. For VCC, blindly inserting a copy when the phi is lowered will
2123       // produce an invalid copy. We can only copy with some kind of compare to
2124       // get a vector boolean result. Insert a regitser bank copy that will be
2125       // correctly lowered to a compare.
2126       MachineIRBuilder B(*MI.getParent()->getParent());
2127 
2128       for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2129         Register SrcReg = MI.getOperand(I).getReg();
2130         const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI);
2131 
2132         if (SrcBank != &AMDGPU::VCCRegBank) {
2133           MachineBasicBlock *SrcMBB = MI.getOperand(I + 1).getMBB();
2134           B.setInsertPt(*SrcMBB, SrcMBB->getFirstTerminator());
2135 
2136           auto Copy = B.buildCopy(LLT::scalar(1), SrcReg);
2137           MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank);
2138           MI.getOperand(I).setReg(Copy.getReg(0));
2139         }
2140       }
2141 
2142       return;
2143     }
2144 
2145     // Phi handling is strange and only considers the bank of the destination.
2146     substituteSimpleCopyRegs(OpdMapper, 0);
2147 
2148     // Promote SGPR/VGPR booleans to s32
2149     MachineFunction *MF = MI.getParent()->getParent();
2150     ApplyRegBankMapping ApplyBank(*this, MRI, DstBank);
2151     GISelObserverWrapper Observer(&ApplyBank);
2152     MachineIRBuilder B(MI);
2153     LegalizerHelper Helper(*MF, Observer, B);
2154 
2155     if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2156       llvm_unreachable("widen scalar should have succeeded");
2157 
2158     return;
2159   }
2160   case AMDGPU::G_ICMP:
2161   case AMDGPU::G_UADDO:
2162   case AMDGPU::G_USUBO:
2163   case AMDGPU::G_UADDE:
2164   case AMDGPU::G_SADDE:
2165   case AMDGPU::G_USUBE:
2166   case AMDGPU::G_SSUBE: {
2167     unsigned BoolDstOp = Opc == AMDGPU::G_ICMP ? 0 : 1;
2168     Register DstReg = MI.getOperand(BoolDstOp).getReg();
2169 
2170     const RegisterBank *DstBank =
2171       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2172     if (DstBank != &AMDGPU::SGPRRegBank)
2173       break;
2174 
2175     const bool HasCarryIn = MI.getNumOperands() == 5;
2176 
2177     // If this is a scalar compare, promote the result to s32, as the selection
2178     // will end up using a copy to a 32-bit vreg.
2179     const LLT S32 = LLT::scalar(32);
2180     Register NewDstReg = MRI.createGenericVirtualRegister(S32);
2181     MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank);
2182     MI.getOperand(BoolDstOp).setReg(NewDstReg);
2183     MachineIRBuilder B(MI);
2184 
2185     if (HasCarryIn) {
2186       Register NewSrcReg = MRI.createGenericVirtualRegister(S32);
2187       MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank);
2188       B.buildZExt(NewSrcReg, MI.getOperand(4).getReg());
2189       MI.getOperand(4).setReg(NewSrcReg);
2190     }
2191 
2192     MachineBasicBlock *MBB = MI.getParent();
2193     B.setInsertPt(*MBB, std::next(MI.getIterator()));
2194 
2195     // If we had a constrained VCC result register, a copy was inserted to VCC
2196     // from SGPR.
2197     SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(0));
2198     if (DefRegs.empty())
2199       DefRegs.push_back(DstReg);
2200     B.buildTrunc(DefRegs[0], NewDstReg);
2201     return;
2202   }
2203   case AMDGPU::G_SELECT: {
2204     Register DstReg = MI.getOperand(0).getReg();
2205     LLT DstTy = MRI.getType(DstReg);
2206 
2207     SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(1));
2208     if (CondRegs.empty())
2209       CondRegs.push_back(MI.getOperand(1).getReg());
2210     else {
2211       assert(CondRegs.size() == 1);
2212     }
2213 
2214     const RegisterBank *CondBank = getRegBank(CondRegs[0], MRI, *TRI);
2215     if (CondBank == &AMDGPU::SGPRRegBank) {
2216       MachineIRBuilder B(MI);
2217       const LLT S32 = LLT::scalar(32);
2218       Register NewCondReg = MRI.createGenericVirtualRegister(S32);
2219       MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2220 
2221       MI.getOperand(1).setReg(NewCondReg);
2222       B.buildZExt(NewCondReg, CondRegs[0]);
2223     }
2224 
2225     if (DstTy.getSizeInBits() != 64)
2226       break;
2227 
2228     MachineIRBuilder B(MI);
2229     LLT HalfTy = getHalfSizedType(DstTy);
2230 
2231     SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2232     SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2233     SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(3));
2234 
2235     // All inputs are SGPRs, nothing special to do.
2236     if (DefRegs.empty()) {
2237       assert(Src1Regs.empty() && Src2Regs.empty());
2238       break;
2239     }
2240 
2241     if (Src1Regs.empty())
2242       split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2243     else {
2244       setRegsToType(MRI, Src1Regs, HalfTy);
2245     }
2246 
2247     if (Src2Regs.empty())
2248       split64BitValueForMapping(B, Src2Regs, HalfTy, MI.getOperand(3).getReg());
2249     else
2250       setRegsToType(MRI, Src2Regs, HalfTy);
2251 
2252     setRegsToType(MRI, DefRegs, HalfTy);
2253 
2254     B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0]);
2255     B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1]);
2256 
2257     MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2258     MI.eraseFromParent();
2259     return;
2260   }
2261   case AMDGPU::G_BRCOND: {
2262     Register CondReg = MI.getOperand(0).getReg();
2263     // FIXME: Should use legalizer helper, but should change bool ext type.
2264     const RegisterBank *CondBank =
2265       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2266 
2267     if (CondBank == &AMDGPU::SGPRRegBank) {
2268       MachineIRBuilder B(MI);
2269       const LLT S32 = LLT::scalar(32);
2270       Register NewCondReg = MRI.createGenericVirtualRegister(S32);
2271       MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2272 
2273       MI.getOperand(0).setReg(NewCondReg);
2274       B.buildZExt(NewCondReg, CondReg);
2275       return;
2276     }
2277 
2278     break;
2279   }
2280   case AMDGPU::G_AND:
2281   case AMDGPU::G_OR:
2282   case AMDGPU::G_XOR: {
2283     // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2284     // there is a VGPR input.
2285     Register DstReg = MI.getOperand(0).getReg();
2286     LLT DstTy = MRI.getType(DstReg);
2287 
2288     if (DstTy.getSizeInBits() == 1) {
2289       const RegisterBank *DstBank =
2290         OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2291       if (DstBank == &AMDGPU::VCCRegBank)
2292         break;
2293 
2294       MachineFunction *MF = MI.getParent()->getParent();
2295       ApplyRegBankMapping ApplyBank(*this, MRI, DstBank);
2296       GISelObserverWrapper Observer(&ApplyBank);
2297       MachineIRBuilder B(MI);
2298       LegalizerHelper Helper(*MF, Observer, B);
2299 
2300       if (Helper.widenScalar(MI, 0, LLT::scalar(32)) !=
2301           LegalizerHelper::Legalized)
2302         llvm_unreachable("widen scalar should have succeeded");
2303       return;
2304     }
2305 
2306     if (DstTy.getSizeInBits() != 64)
2307       break;
2308 
2309     LLT HalfTy = getHalfSizedType(DstTy);
2310     SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2311     SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(1));
2312     SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2313 
2314     // All inputs are SGPRs, nothing special to do.
2315     if (DefRegs.empty()) {
2316       assert(Src0Regs.empty() && Src1Regs.empty());
2317       break;
2318     }
2319 
2320     assert(DefRegs.size() == 2);
2321     assert(Src0Regs.size() == Src1Regs.size() &&
2322            (Src0Regs.empty() || Src0Regs.size() == 2));
2323 
2324     // Depending on where the source registers came from, the generic code may
2325     // have decided to split the inputs already or not. If not, we still need to
2326     // extract the values.
2327     MachineIRBuilder B(MI);
2328 
2329     if (Src0Regs.empty())
2330       split64BitValueForMapping(B, Src0Regs, HalfTy, MI.getOperand(1).getReg());
2331     else
2332       setRegsToType(MRI, Src0Regs, HalfTy);
2333 
2334     if (Src1Regs.empty())
2335       split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2336     else
2337       setRegsToType(MRI, Src1Regs, HalfTy);
2338 
2339     setRegsToType(MRI, DefRegs, HalfTy);
2340 
2341     B.buildInstr(Opc, {DefRegs[0]}, {Src0Regs[0], Src1Regs[0]});
2342     B.buildInstr(Opc, {DefRegs[1]}, {Src0Regs[1], Src1Regs[1]});
2343 
2344     MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2345     MI.eraseFromParent();
2346     return;
2347   }
2348   case AMDGPU::G_ADD:
2349   case AMDGPU::G_SUB:
2350   case AMDGPU::G_MUL:
2351   case AMDGPU::G_SHL:
2352   case AMDGPU::G_LSHR:
2353   case AMDGPU::G_ASHR: {
2354     Register DstReg = MI.getOperand(0).getReg();
2355     LLT DstTy = MRI.getType(DstReg);
2356 
2357     // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2358     // Packed 16-bit operations need to be scalarized and promoted.
2359     if (DstTy != LLT::scalar(16) && DstTy != LLT::vector(2, 16))
2360       break;
2361 
2362     const RegisterBank *DstBank =
2363       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2364     if (DstBank == &AMDGPU::VGPRRegBank)
2365       break;
2366 
2367     const LLT S32 = LLT::scalar(32);
2368     MachineBasicBlock *MBB = MI.getParent();
2369     MachineFunction *MF = MBB->getParent();
2370     MachineIRBuilder B(MI);
2371     ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2372     GISelObserverWrapper Observer(&ApplySALU);
2373 
2374     if (DstTy.isVector()) {
2375       B.setChangeObserver(Observer);
2376 
2377       Register WideSrc0Lo, WideSrc0Hi;
2378       Register WideSrc1Lo, WideSrc1Hi;
2379 
2380       std::tie(WideSrc0Lo, WideSrc0Hi)
2381         = unpackV2S16ToS32(B, MI.getOperand(1).getReg(), AMDGPU::G_ANYEXT);
2382       std::tie(WideSrc1Lo, WideSrc1Hi)
2383         = unpackV2S16ToS32(B, MI.getOperand(2).getReg(), AMDGPU::G_ANYEXT);
2384       auto Lo = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Lo, WideSrc1Lo});
2385       auto Hi = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Hi, WideSrc1Hi});
2386       B.buildBuildVectorTrunc(DstReg, {Lo.getReg(0), Hi.getReg(0)});
2387       MI.eraseFromParent();
2388     } else {
2389       LegalizerHelper Helper(*MF, Observer, B);
2390 
2391       if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2392         llvm_unreachable("widen scalar should have succeeded");
2393 
2394       // FIXME: s16 shift amounts should be legal.
2395       if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR ||
2396           Opc == AMDGPU::G_ASHR) {
2397         B.setInsertPt(*MBB, MI.getIterator());
2398         if (Helper.widenScalar(MI, 1, S32) != LegalizerHelper::Legalized)
2399           llvm_unreachable("widen scalar should have succeeded");
2400       }
2401     }
2402 
2403     return;
2404   }
2405   case AMDGPU::G_SMIN:
2406   case AMDGPU::G_SMAX:
2407   case AMDGPU::G_UMIN:
2408   case AMDGPU::G_UMAX: {
2409     Register DstReg = MI.getOperand(0).getReg();
2410     const RegisterBank *DstBank =
2411       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2412     if (DstBank == &AMDGPU::VGPRRegBank)
2413       break;
2414 
2415     MachineFunction *MF = MI.getParent()->getParent();
2416     MachineIRBuilder B(MI);
2417 
2418     // Turn scalar min/max into a compare and select.
2419     LLT Ty = MRI.getType(DstReg);
2420     const LLT S32 = LLT::scalar(32);
2421     const LLT S16 = LLT::scalar(16);
2422     const LLT V2S16 = LLT::vector(2, 16);
2423 
2424     if (Ty == V2S16) {
2425       ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2426       GISelObserverWrapper Observer(&ApplySALU);
2427       B.setChangeObserver(Observer);
2428 
2429       // Need to widen to s32, and expand as cmp + select, and avoid producing
2430       // illegal vector extends or unmerges that would need further
2431       // legalization.
2432       //
2433       // TODO: Should we just readfirstlane? That should probably be handled
2434       // with a UniformVGPR register bank that wouldn't need special
2435       // consideration here.
2436 
2437       Register Dst = MI.getOperand(0).getReg();
2438       Register Src0 = MI.getOperand(1).getReg();
2439       Register Src1 = MI.getOperand(2).getReg();
2440 
2441       Register WideSrc0Lo, WideSrc0Hi;
2442       Register WideSrc1Lo, WideSrc1Hi;
2443 
2444       unsigned ExtendOp = minMaxToExtend(MI.getOpcode());
2445 
2446       std::tie(WideSrc0Lo, WideSrc0Hi) = unpackV2S16ToS32(B, Src0, ExtendOp);
2447       std::tie(WideSrc1Lo, WideSrc1Hi) = unpackV2S16ToS32(B, Src1, ExtendOp);
2448 
2449       Register Lo = MRI.createGenericVirtualRegister(S32);
2450       Register Hi = MRI.createGenericVirtualRegister(S32);
2451       const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
2452       buildExpandedScalarMinMax(B, Pred, Lo, WideSrc0Lo, WideSrc1Lo);
2453       buildExpandedScalarMinMax(B, Pred, Hi, WideSrc0Hi, WideSrc1Hi);
2454 
2455       B.buildBuildVectorTrunc(Dst, {Lo, Hi});
2456       MI.eraseFromParent();
2457     } else if (Ty == S16) {
2458       ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2459       GISelObserverWrapper Observer(&ApplySALU);
2460       LegalizerHelper Helper(*MF, Observer, B);
2461 
2462       // Need to widen to s32, and expand as cmp + select.
2463       if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2464         llvm_unreachable("widenScalar should have succeeded");
2465 
2466       // FIXME: This is relying on widenScalar leaving MI in place.
2467       lowerScalarMinMax(B, MI);
2468     } else
2469       lowerScalarMinMax(B, MI);
2470 
2471     return;
2472   }
2473   case AMDGPU::G_SEXT_INREG: {
2474     SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1));
2475     if (SrcRegs.empty())
2476       break; // Nothing to repair
2477 
2478     const LLT S32 = LLT::scalar(32);
2479     MachineIRBuilder B(MI);
2480     ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank);
2481     GISelObserverWrapper Observer(&O);
2482     B.setChangeObserver(Observer);
2483 
2484     // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs
2485     // we would need to further expand, and doesn't let us directly set the
2486     // result registers.
2487     SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2488 
2489     int Amt = MI.getOperand(2).getImm();
2490     if (Amt <= 32) {
2491       if (Amt == 32) {
2492         // The low bits are unchanged.
2493         B.buildCopy(DstRegs[0], SrcRegs[0]);
2494       } else {
2495         // Extend in the low bits and propagate the sign bit to the high half.
2496         B.buildSExtInReg(DstRegs[0], SrcRegs[0], Amt);
2497       }
2498 
2499       B.buildAShr(DstRegs[1], DstRegs[0], B.buildConstant(S32, 31));
2500     } else {
2501       // The low bits are unchanged, and extend in the high bits.
2502       B.buildCopy(DstRegs[0], SrcRegs[0]);
2503       B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32);
2504     }
2505 
2506     Register DstReg = MI.getOperand(0).getReg();
2507     MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2508     MI.eraseFromParent();
2509     return;
2510   }
2511   case AMDGPU::G_CTPOP:
2512   case AMDGPU::G_CTLZ_ZERO_UNDEF:
2513   case AMDGPU::G_CTTZ_ZERO_UNDEF: {
2514     MachineIRBuilder B(MI);
2515     MachineFunction &MF = B.getMF();
2516 
2517     const RegisterBank *DstBank =
2518       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2519     if (DstBank == &AMDGPU::SGPRRegBank)
2520       break;
2521 
2522     Register SrcReg = MI.getOperand(1).getReg();
2523     const LLT S32 = LLT::scalar(32);
2524     LLT Ty = MRI.getType(SrcReg);
2525     if (Ty == S32)
2526       break;
2527 
2528     ApplyRegBankMapping ApplyVALU(*this, MRI, &AMDGPU::VGPRRegBank);
2529     GISelObserverWrapper Observer(&ApplyVALU);
2530     LegalizerHelper Helper(MF, Observer, B);
2531 
2532     if (Helper.narrowScalar(MI, 1, S32) != LegalizerHelper::Legalized)
2533       llvm_unreachable("narrowScalar should have succeeded");
2534     return;
2535   }
2536   case AMDGPU::G_SEXT:
2537   case AMDGPU::G_ZEXT:
2538   case AMDGPU::G_ANYEXT: {
2539     Register SrcReg = MI.getOperand(1).getReg();
2540     LLT SrcTy = MRI.getType(SrcReg);
2541     const bool Signed = Opc == AMDGPU::G_SEXT;
2542 
2543     assert(empty(OpdMapper.getVRegs(1)));
2544 
2545     MachineIRBuilder B(MI);
2546     const RegisterBank *SrcBank =
2547       OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2548 
2549     Register DstReg = MI.getOperand(0).getReg();
2550     LLT DstTy = MRI.getType(DstReg);
2551     if (DstTy.isScalar() &&
2552         SrcBank != &AMDGPU::SGPRRegBank &&
2553         SrcBank != &AMDGPU::VCCRegBank &&
2554         // FIXME: Should handle any type that round to s64 when irregular
2555         // breakdowns supported.
2556         DstTy.getSizeInBits() == 64 &&
2557         SrcTy.getSizeInBits() <= 32) {
2558       SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2559 
2560       // Extend to 32-bit, and then extend the low half.
2561       if (Signed) {
2562         // TODO: Should really be buildSExtOrCopy
2563         B.buildSExtOrTrunc(DefRegs[0], SrcReg);
2564       } else if (Opc == AMDGPU::G_ZEXT) {
2565         B.buildZExtOrTrunc(DefRegs[0], SrcReg);
2566       } else {
2567         B.buildAnyExtOrTrunc(DefRegs[0], SrcReg);
2568       }
2569 
2570       extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank);
2571       MRI.setRegBank(DstReg, *SrcBank);
2572       MI.eraseFromParent();
2573       return;
2574     }
2575 
2576     if (SrcTy != LLT::scalar(1))
2577       return;
2578 
2579     // It is not legal to have a legalization artifact with a VCC source. Rather
2580     // than introducing a copy, insert the select we would have to select the
2581     // copy to.
2582     if (SrcBank == &AMDGPU::VCCRegBank) {
2583       SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2584 
2585       const RegisterBank *DstBank = &AMDGPU::VGPRRegBank;
2586 
2587       unsigned DstSize = DstTy.getSizeInBits();
2588       // 64-bit select is SGPR only
2589       const bool UseSel64 = DstSize > 32 &&
2590         SrcBank->getID() == AMDGPU::SGPRRegBankID;
2591 
2592       // TODO: Should s16 select be legal?
2593       LLT SelType = UseSel64 ? LLT::scalar(64) : LLT::scalar(32);
2594       auto True = B.buildConstant(SelType, Signed ? -1 : 1);
2595       auto False = B.buildConstant(SelType, 0);
2596 
2597       MRI.setRegBank(True.getReg(0), *DstBank);
2598       MRI.setRegBank(False.getReg(0), *DstBank);
2599       MRI.setRegBank(DstReg, *DstBank);
2600 
2601       if (DstSize > 32) {
2602         B.buildSelect(DefRegs[0], SrcReg, True, False);
2603         extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank, true);
2604       } else if (DstSize < 32) {
2605         auto Sel = B.buildSelect(SelType, SrcReg, True, False);
2606         MRI.setRegBank(Sel.getReg(0), *DstBank);
2607         B.buildTrunc(DstReg, Sel);
2608       } else {
2609         B.buildSelect(DstReg, SrcReg, True, False);
2610       }
2611 
2612       MI.eraseFromParent();
2613       return;
2614     }
2615 
2616     break;
2617   }
2618   case AMDGPU::G_BUILD_VECTOR:
2619   case AMDGPU::G_BUILD_VECTOR_TRUNC: {
2620     Register DstReg = MI.getOperand(0).getReg();
2621     LLT DstTy = MRI.getType(DstReg);
2622     if (DstTy != LLT::vector(2, 16))
2623       break;
2624 
2625     assert(MI.getNumOperands() == 3 && OpdMapper.getVRegs(0).empty());
2626     substituteSimpleCopyRegs(OpdMapper, 1);
2627     substituteSimpleCopyRegs(OpdMapper, 2);
2628 
2629     const RegisterBank *DstBank =
2630       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2631     if (DstBank == &AMDGPU::SGPRRegBank)
2632       break; // Can use S_PACK_* instructions.
2633 
2634     MachineIRBuilder B(MI);
2635 
2636     Register Lo = MI.getOperand(1).getReg();
2637     Register Hi = MI.getOperand(2).getReg();
2638     const LLT S32 = LLT::scalar(32);
2639 
2640     const RegisterBank *BankLo =
2641       OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2642     const RegisterBank *BankHi =
2643       OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2644 
2645     Register ZextLo;
2646     Register ShiftHi;
2647 
2648     if (Opc == AMDGPU::G_BUILD_VECTOR) {
2649       ZextLo = B.buildZExt(S32, Lo).getReg(0);
2650       MRI.setRegBank(ZextLo, *BankLo);
2651 
2652       Register ZextHi = B.buildZExt(S32, Hi).getReg(0);
2653       MRI.setRegBank(ZextHi, *BankHi);
2654 
2655       auto ShiftAmt = B.buildConstant(S32, 16);
2656       MRI.setRegBank(ShiftAmt.getReg(0), *BankHi);
2657 
2658       ShiftHi = B.buildShl(S32, ZextHi, ShiftAmt).getReg(0);
2659       MRI.setRegBank(ShiftHi, *BankHi);
2660     } else {
2661       Register MaskLo = B.buildConstant(S32, 0xffff).getReg(0);
2662       MRI.setRegBank(MaskLo, *BankLo);
2663 
2664       auto ShiftAmt = B.buildConstant(S32, 16);
2665       MRI.setRegBank(ShiftAmt.getReg(0), *BankHi);
2666 
2667       ShiftHi = B.buildShl(S32, Hi, ShiftAmt).getReg(0);
2668       MRI.setRegBank(ShiftHi, *BankHi);
2669 
2670       ZextLo = B.buildAnd(S32, Lo, MaskLo).getReg(0);
2671       MRI.setRegBank(ZextLo, *BankLo);
2672     }
2673 
2674     auto Or = B.buildOr(S32, ZextLo, ShiftHi);
2675     MRI.setRegBank(Or.getReg(0), *DstBank);
2676 
2677     B.buildBitcast(DstReg, Or);
2678     MI.eraseFromParent();
2679     return;
2680   }
2681   case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2682     SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2683 
2684     assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2685 
2686     Register DstReg = MI.getOperand(0).getReg();
2687     Register SrcReg = MI.getOperand(1).getReg();
2688 
2689     const LLT S32 = LLT::scalar(32);
2690     LLT DstTy = MRI.getType(DstReg);
2691     LLT SrcTy = MRI.getType(SrcReg);
2692 
2693     if (foldExtractEltToCmpSelect(MI, MRI, OpdMapper))
2694       return;
2695 
2696     MachineIRBuilder B(MI);
2697 
2698     const ValueMapping &DstMapping
2699       = OpdMapper.getInstrMapping().getOperandMapping(0);
2700     const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank;
2701     const RegisterBank *SrcBank =
2702       OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2703     const RegisterBank *IdxBank =
2704         OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2705 
2706     Register BaseIdxReg;
2707     unsigned ConstOffset;
2708     MachineInstr *OffsetDef;
2709     std::tie(BaseIdxReg, ConstOffset, OffsetDef) =
2710         AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg());
2711 
2712     // See if the index is an add of a constant which will be foldable by moving
2713     // the base register of the index later if this is going to be executed in a
2714     // waterfall loop. This is essentially to reassociate the add of a constant
2715     // with the readfirstlane.
2716     bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2717                                    ConstOffset > 0 &&
2718                                    ConstOffset < SrcTy.getNumElements();
2719 
2720     // Move the base register. We'll re-insert the add later.
2721     if (ShouldMoveIndexIntoLoop)
2722       MI.getOperand(2).setReg(BaseIdxReg);
2723 
2724     // If this is a VGPR result only because the index was a VGPR result, the
2725     // actual indexing will be done on the SGPR source vector, which will
2726     // produce a scalar result. We need to copy to the VGPR result inside the
2727     // waterfall loop.
2728     const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2729                                 SrcBank == &AMDGPU::SGPRRegBank;
2730     if (DstRegs.empty()) {
2731       applyDefaultMapping(OpdMapper);
2732 
2733       executeInWaterfallLoop(MI, MRI, { 2 });
2734 
2735       if (NeedCopyToVGPR) {
2736         // We don't want a phi for this temporary reg.
2737         Register TmpReg = MRI.createGenericVirtualRegister(DstTy);
2738         MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank);
2739         MI.getOperand(0).setReg(TmpReg);
2740         B.setInsertPt(*MI.getParent(), ++MI.getIterator());
2741 
2742         // Use a v_mov_b32 here to make the exec dependency explicit.
2743         buildVCopy(B, DstReg, TmpReg);
2744       }
2745 
2746       // Re-insert the constant offset add inside the waterfall loop.
2747       if (ShouldMoveIndexIntoLoop)
2748         reinsertVectorIndexAdd(B, MI, 2, ConstOffset);
2749 
2750       return;
2751     }
2752 
2753     assert(DstTy.getSizeInBits() == 64);
2754 
2755     LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32);
2756 
2757     auto CastSrc = B.buildBitcast(Vec32, SrcReg);
2758     auto One = B.buildConstant(S32, 1);
2759 
2760     MachineBasicBlock::iterator MII = MI.getIterator();
2761 
2762     // Split the vector index into 32-bit pieces. Prepare to move all of the
2763     // new instructions into a waterfall loop if necessary.
2764     //
2765     // Don't put the bitcast or constant in the loop.
2766     MachineInstrSpan Span(MII, &B.getMBB());
2767 
2768     // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2769     auto IdxLo = B.buildShl(S32, BaseIdxReg, One);
2770     auto IdxHi = B.buildAdd(S32, IdxLo, One);
2771 
2772     auto Extract0 = B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo);
2773     auto Extract1 = B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi);
2774 
2775     MRI.setRegBank(DstReg, *DstBank);
2776     MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2777     MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
2778     MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
2779     MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
2780 
2781     SmallSet<Register, 4> OpsToWaterfall;
2782     if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 2 })) {
2783       MI.eraseFromParent();
2784       return;
2785     }
2786 
2787     // Remove the original instruction to avoid potentially confusing the
2788     // waterfall loop logic.
2789     B.setInstr(*Span.begin());
2790     MI.eraseFromParent();
2791     executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
2792                            OpsToWaterfall, MRI);
2793 
2794     if (NeedCopyToVGPR) {
2795       MachineBasicBlock *LoopBB = Extract1->getParent();
2796       Register TmpReg0 = MRI.createGenericVirtualRegister(S32);
2797       Register TmpReg1 = MRI.createGenericVirtualRegister(S32);
2798       MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank);
2799       MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank);
2800 
2801       Extract0->getOperand(0).setReg(TmpReg0);
2802       Extract1->getOperand(0).setReg(TmpReg1);
2803 
2804       B.setInsertPt(*LoopBB, ++Extract1->getIterator());
2805 
2806       buildVCopy(B, DstRegs[0], TmpReg0);
2807       buildVCopy(B, DstRegs[1], TmpReg1);
2808     }
2809 
2810     if (ShouldMoveIndexIntoLoop)
2811       reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset);
2812 
2813     return;
2814   }
2815   case AMDGPU::G_INSERT_VECTOR_ELT: {
2816     SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2817 
2818     Register DstReg = MI.getOperand(0).getReg();
2819     LLT VecTy = MRI.getType(DstReg);
2820 
2821     assert(OpdMapper.getVRegs(0).empty());
2822     assert(OpdMapper.getVRegs(3).empty());
2823 
2824     if (substituteSimpleCopyRegs(OpdMapper, 1))
2825       MRI.setType(MI.getOperand(1).getReg(), VecTy);
2826 
2827     if (foldInsertEltToCmpSelect(MI, MRI, OpdMapper))
2828       return;
2829 
2830     const RegisterBank *IdxBank =
2831       OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank;
2832 
2833     Register SrcReg = MI.getOperand(1).getReg();
2834     Register InsReg = MI.getOperand(2).getReg();
2835     LLT InsTy = MRI.getType(InsReg);
2836     (void)InsTy;
2837 
2838     Register BaseIdxReg;
2839     unsigned ConstOffset;
2840     MachineInstr *OffsetDef;
2841     std::tie(BaseIdxReg, ConstOffset, OffsetDef) =
2842       AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(3).getReg());
2843 
2844     // See if the index is an add of a constant which will be foldable by moving
2845     // the base register of the index later if this is going to be executed in a
2846     // waterfall loop. This is essentially to reassociate the add of a constant
2847     // with the readfirstlane.
2848     bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2849       ConstOffset > 0 &&
2850       ConstOffset < VecTy.getNumElements();
2851 
2852     // Move the base register. We'll re-insert the add later.
2853     if (ShouldMoveIndexIntoLoop)
2854       MI.getOperand(3).setReg(BaseIdxReg);
2855 
2856 
2857     if (InsRegs.empty()) {
2858       executeInWaterfallLoop(MI, MRI, { 3 });
2859 
2860       // Re-insert the constant offset add inside the waterfall loop.
2861       if (ShouldMoveIndexIntoLoop) {
2862         MachineIRBuilder B(MI);
2863         reinsertVectorIndexAdd(B, MI, 3, ConstOffset);
2864       }
2865 
2866       return;
2867     }
2868 
2869 
2870     assert(InsTy.getSizeInBits() == 64);
2871 
2872     const LLT S32 = LLT::scalar(32);
2873     LLT Vec32 = LLT::vector(2 * VecTy.getNumElements(), 32);
2874 
2875     MachineIRBuilder B(MI);
2876     auto CastSrc = B.buildBitcast(Vec32, SrcReg);
2877     auto One = B.buildConstant(S32, 1);
2878 
2879     // Split the vector index into 32-bit pieces. Prepare to move all of the
2880     // new instructions into a waterfall loop if necessary.
2881     //
2882     // Don't put the bitcast or constant in the loop.
2883     MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB());
2884 
2885     // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2886     auto IdxLo = B.buildShl(S32, BaseIdxReg, One);
2887     auto IdxHi = B.buildAdd(S32, IdxLo, One);
2888 
2889     auto InsLo = B.buildInsertVectorElement(Vec32, CastSrc, InsRegs[0], IdxLo);
2890     auto InsHi = B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi);
2891 
2892     const RegisterBank *DstBank =
2893       OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2894     const RegisterBank *SrcBank =
2895       OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2896     const RegisterBank *InsSrcBank =
2897       OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2898 
2899     MRI.setRegBank(InsReg, *InsSrcBank);
2900     MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2901     MRI.setRegBank(InsLo.getReg(0), *DstBank);
2902     MRI.setRegBank(InsHi.getReg(0), *DstBank);
2903     MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
2904     MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
2905     MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
2906 
2907 
2908     SmallSet<Register, 4> OpsToWaterfall;
2909     if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 3 })) {
2910       B.setInsertPt(B.getMBB(), MI);
2911       B.buildBitcast(DstReg, InsHi);
2912       MI.eraseFromParent();
2913       return;
2914     }
2915 
2916     B.setInstr(*Span.begin());
2917     MI.eraseFromParent();
2918 
2919     // Figure out the point after the waterfall loop before mangling the control
2920     // flow.
2921     executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
2922                            OpsToWaterfall, MRI);
2923 
2924     // The insertion point is now right after the original instruction.
2925     //
2926     // Keep the bitcast to the original vector type out of the loop. Doing this
2927     // saved an extra phi we don't need inside the loop.
2928     B.buildBitcast(DstReg, InsHi);
2929 
2930     // Re-insert the constant offset add inside the waterfall loop.
2931     if (ShouldMoveIndexIntoLoop)
2932       reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset);
2933 
2934     return;
2935   }
2936   case AMDGPU::G_AMDGPU_BUFFER_LOAD:
2937   case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
2938   case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
2939   case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
2940   case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
2941   case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
2942   case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
2943   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
2944   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
2945   case AMDGPU::G_AMDGPU_BUFFER_STORE:
2946   case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
2947   case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
2948   case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
2949   case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
2950   case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
2951   case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
2952     applyDefaultMapping(OpdMapper);
2953     executeInWaterfallLoop(MI, MRI, {1, 4});
2954     return;
2955   }
2956   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
2957   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
2958   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
2959   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
2960   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
2961   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
2962   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
2963   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
2964   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
2965   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
2966   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
2967   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: {
2968     applyDefaultMapping(OpdMapper);
2969     executeInWaterfallLoop(MI, MRI, {2, 5});
2970     return;
2971   }
2972   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: {
2973     applyDefaultMapping(OpdMapper);
2974     executeInWaterfallLoop(MI, MRI, {1, 4});
2975     return;
2976   }
2977   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
2978     applyDefaultMapping(OpdMapper);
2979     executeInWaterfallLoop(MI, MRI, {3, 6});
2980     return;
2981   }
2982   case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: {
2983     applyMappingSBufferLoad(OpdMapper);
2984     return;
2985   }
2986   case AMDGPU::G_INTRINSIC: {
2987     switch (MI.getIntrinsicID()) {
2988     case Intrinsic::amdgcn_readlane: {
2989       substituteSimpleCopyRegs(OpdMapper, 2);
2990 
2991       assert(OpdMapper.getVRegs(0).empty());
2992       assert(OpdMapper.getVRegs(3).empty());
2993 
2994       // Make sure the index is an SGPR. It doesn't make sense to run this in a
2995       // waterfall loop, so assume it's a uniform value.
2996       constrainOpWithReadfirstlane(MI, MRI, 3); // Index
2997       return;
2998     }
2999     case Intrinsic::amdgcn_writelane: {
3000       assert(OpdMapper.getVRegs(0).empty());
3001       assert(OpdMapper.getVRegs(2).empty());
3002       assert(OpdMapper.getVRegs(3).empty());
3003 
3004       substituteSimpleCopyRegs(OpdMapper, 4); // VGPR input val
3005       constrainOpWithReadfirstlane(MI, MRI, 2); // Source value
3006       constrainOpWithReadfirstlane(MI, MRI, 3); // Index
3007       return;
3008     }
3009     case Intrinsic::amdgcn_interp_p1:
3010     case Intrinsic::amdgcn_interp_p2:
3011     case Intrinsic::amdgcn_interp_mov:
3012     case Intrinsic::amdgcn_interp_p1_f16:
3013     case Intrinsic::amdgcn_interp_p2_f16: {
3014       applyDefaultMapping(OpdMapper);
3015 
3016       // Readlane for m0 value, which is always the last operand.
3017       // FIXME: Should this be a waterfall loop instead?
3018       constrainOpWithReadfirstlane(MI, MRI, MI.getNumOperands() - 1); // Index
3019       return;
3020     }
3021     case Intrinsic::amdgcn_permlane16:
3022     case Intrinsic::amdgcn_permlanex16: {
3023       // Doing a waterfall loop over these wouldn't make any sense.
3024       substituteSimpleCopyRegs(OpdMapper, 2);
3025       substituteSimpleCopyRegs(OpdMapper, 3);
3026       constrainOpWithReadfirstlane(MI, MRI, 4);
3027       constrainOpWithReadfirstlane(MI, MRI, 5);
3028       return;
3029     }
3030     case Intrinsic::amdgcn_sbfe:
3031       applyMappingBFEIntrinsic(OpdMapper, true);
3032       return;
3033     case Intrinsic::amdgcn_ubfe:
3034       applyMappingBFEIntrinsic(OpdMapper, false);
3035       return;
3036     case Intrinsic::amdgcn_ballot:
3037       // Use default handling and insert copy to vcc source.
3038       break;
3039     }
3040     break;
3041   }
3042   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3043   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
3044     const AMDGPU::RsrcIntrinsic *RSrcIntrin
3045       = AMDGPU::lookupRsrcIntrinsic(MI.getIntrinsicID());
3046     assert(RSrcIntrin && RSrcIntrin->IsImage);
3047     // Non-images can have complications from operands that allow both SGPR
3048     // and VGPR. For now it's too complicated to figure out the final opcode
3049     // to derive the register bank from the MCInstrDesc.
3050     applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
3051     return;
3052   }
3053   case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
3054     auto IntrID = MI.getIntrinsicID();
3055     switch (IntrID) {
3056     case Intrinsic::amdgcn_ds_ordered_add:
3057     case Intrinsic::amdgcn_ds_ordered_swap: {
3058       // This is only allowed to execute with 1 lane, so readfirstlane is safe.
3059       assert(OpdMapper.getVRegs(0).empty());
3060       substituteSimpleCopyRegs(OpdMapper, 3);
3061       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3062       return;
3063     }
3064     case Intrinsic::amdgcn_ds_gws_init:
3065     case Intrinsic::amdgcn_ds_gws_barrier:
3066     case Intrinsic::amdgcn_ds_gws_sema_br: {
3067       // Only the first lane is executes, so readfirstlane is safe.
3068       substituteSimpleCopyRegs(OpdMapper, 1);
3069       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3070       return;
3071     }
3072     case Intrinsic::amdgcn_ds_gws_sema_v:
3073     case Intrinsic::amdgcn_ds_gws_sema_p:
3074     case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3075       // Only the first lane is executes, so readfirstlane is safe.
3076       constrainOpWithReadfirstlane(MI, MRI, 1); // M0
3077       return;
3078     }
3079     case Intrinsic::amdgcn_ds_append:
3080     case Intrinsic::amdgcn_ds_consume: {
3081       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3082       return;
3083     }
3084     case Intrinsic::amdgcn_s_sendmsg:
3085     case Intrinsic::amdgcn_s_sendmsghalt: {
3086       // FIXME: Should this use a waterfall loop?
3087       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3088       return;
3089     }
3090     case Intrinsic::amdgcn_s_setreg: {
3091       constrainOpWithReadfirstlane(MI, MRI, 2);
3092       return;
3093     }
3094     default: {
3095       if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3096               AMDGPU::lookupRsrcIntrinsic(IntrID)) {
3097         // Non-images can have complications from operands that allow both SGPR
3098         // and VGPR. For now it's too complicated to figure out the final opcode
3099         // to derive the register bank from the MCInstrDesc.
3100         if (RSrcIntrin->IsImage) {
3101           applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
3102           return;
3103         }
3104       }
3105 
3106       break;
3107     }
3108     }
3109     break;
3110   }
3111   case AMDGPU::G_LOAD:
3112   case AMDGPU::G_ZEXTLOAD:
3113   case AMDGPU::G_SEXTLOAD: {
3114     if (applyMappingLoad(MI, OpdMapper, MRI))
3115       return;
3116     break;
3117   }
3118   case AMDGPU::G_DYN_STACKALLOC:
3119     applyMappingDynStackAlloc(MI, OpdMapper, MRI);
3120     return;
3121   default:
3122     break;
3123   }
3124 
3125   return applyDefaultMapping(OpdMapper);
3126 }
3127 
3128 // vgpr, sgpr -> vgpr
3129 // vgpr, agpr -> vgpr
3130 // agpr, agpr -> agpr
3131 // agpr, sgpr -> vgpr
3132 static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
3133   if (RB0 == AMDGPU::InvalidRegBankID)
3134     return RB1;
3135   if (RB1 == AMDGPU::InvalidRegBankID)
3136     return RB0;
3137 
3138   if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID)
3139     return AMDGPU::SGPRRegBankID;
3140 
3141   if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID)
3142     return AMDGPU::AGPRRegBankID;
3143 
3144   return AMDGPU::VGPRRegBankID;
3145 }
3146 
3147 static unsigned regBankBoolUnion(unsigned RB0, unsigned RB1) {
3148   if (RB0 == AMDGPU::InvalidRegBankID)
3149     return RB1;
3150   if (RB1 == AMDGPU::InvalidRegBankID)
3151     return RB0;
3152 
3153   // vcc, vcc -> vcc
3154   // vcc, sgpr -> vcc
3155   // vcc, vgpr -> vcc
3156   if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID)
3157     return AMDGPU::VCCRegBankID;
3158 
3159   // vcc, vgpr -> vgpr
3160   return regBankUnion(RB0, RB1);
3161 }
3162 
3163 unsigned AMDGPURegisterBankInfo::getMappingType(const MachineRegisterInfo &MRI,
3164                                                 const MachineInstr &MI) const {
3165   unsigned RegBank = AMDGPU::InvalidRegBankID;
3166 
3167   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3168     if (!MI.getOperand(i).isReg())
3169       continue;
3170     Register Reg = MI.getOperand(i).getReg();
3171     if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
3172       RegBank = regBankUnion(RegBank, Bank->getID());
3173       if (RegBank == AMDGPU::VGPRRegBankID)
3174         break;
3175     }
3176   }
3177 
3178   return RegBank;
3179 }
3180 
3181 bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
3182   const MachineFunction &MF = *MI.getParent()->getParent();
3183   const MachineRegisterInfo &MRI = MF.getRegInfo();
3184   for (unsigned i = 0, e = MI.getNumOperands();i != e; ++i) {
3185     if (!MI.getOperand(i).isReg())
3186       continue;
3187     Register Reg = MI.getOperand(i).getReg();
3188     if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
3189       if (Bank->getID() != AMDGPU::SGPRRegBankID)
3190         return false;
3191     }
3192   }
3193   return true;
3194 }
3195 
3196 const RegisterBankInfo::InstructionMapping &
3197 AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
3198   const MachineFunction &MF = *MI.getParent()->getParent();
3199   const MachineRegisterInfo &MRI = MF.getRegInfo();
3200   SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3201 
3202   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3203     const MachineOperand &SrcOp = MI.getOperand(i);
3204     if (!SrcOp.isReg())
3205       continue;
3206 
3207     unsigned Size = getSizeInBits(SrcOp.getReg(), MRI, *TRI);
3208     OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3209   }
3210   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3211                                MI.getNumOperands());
3212 }
3213 
3214 const RegisterBankInfo::InstructionMapping &
3215 AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
3216   const MachineFunction &MF = *MI.getParent()->getParent();
3217   const MachineRegisterInfo &MRI = MF.getRegInfo();
3218   SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3219 
3220   // Even though we technically could use SGPRs, this would require knowledge of
3221   // the constant bus restriction. Force all sources to VGPR (except for VCC).
3222   //
3223   // TODO: Unary ops are trivially OK, so accept SGPRs?
3224   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3225     const MachineOperand &Src = MI.getOperand(i);
3226     if (!Src.isReg())
3227       continue;
3228 
3229     unsigned Size = getSizeInBits(Src.getReg(), MRI, *TRI);
3230     unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
3231     OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
3232   }
3233 
3234   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3235                                MI.getNumOperands());
3236 }
3237 
3238 const RegisterBankInfo::InstructionMapping &
3239 AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
3240   const MachineFunction &MF = *MI.getParent()->getParent();
3241   const MachineRegisterInfo &MRI = MF.getRegInfo();
3242   SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3243 
3244   for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
3245     const MachineOperand &Op = MI.getOperand(I);
3246     if (!Op.isReg())
3247       continue;
3248 
3249     unsigned Size = getSizeInBits(Op.getReg(), MRI, *TRI);
3250     OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3251   }
3252 
3253   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3254                                MI.getNumOperands());
3255 }
3256 
3257 const RegisterBankInfo::InstructionMapping &
3258 AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
3259                                         const MachineInstr &MI,
3260                                         int RsrcIdx) const {
3261   // The reported argument index is relative to the IR intrinsic call arguments,
3262   // so we need to shift by the number of defs and the intrinsic ID.
3263   RsrcIdx += MI.getNumExplicitDefs() + 1;
3264 
3265   const int NumOps = MI.getNumOperands();
3266   SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
3267 
3268   // TODO: Should packed/unpacked D16 difference be reported here as part of
3269   // the value mapping?
3270   for (int I = 0; I != NumOps; ++I) {
3271     if (!MI.getOperand(I).isReg())
3272       continue;
3273 
3274     Register OpReg = MI.getOperand(I).getReg();
3275     // We replace some dead address operands with $noreg
3276     if (!OpReg)
3277       continue;
3278 
3279     unsigned Size = getSizeInBits(OpReg, MRI, *TRI);
3280 
3281     // FIXME: Probably need a new intrinsic register bank searchable table to
3282     // handle arbitrary intrinsics easily.
3283     //
3284     // If this has a sampler, it immediately follows rsrc.
3285     const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
3286 
3287     if (MustBeSGPR) {
3288       // If this must be an SGPR, so we must report whatever it is as legal.
3289       unsigned NewBank = getRegBankID(OpReg, MRI, AMDGPU::SGPRRegBankID);
3290       OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size);
3291     } else {
3292       // Some operands must be VGPR, and these are easy to copy to.
3293       OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3294     }
3295   }
3296 
3297   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), NumOps);
3298 }
3299 
3300 /// Return the mapping for a pointer arugment.
3301 const RegisterBankInfo::ValueMapping *
3302 AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
3303                                               Register PtrReg) const {
3304   LLT PtrTy = MRI.getType(PtrReg);
3305   unsigned Size = PtrTy.getSizeInBits();
3306   if (Subtarget.useFlatForGlobal() ||
3307       !AMDGPU::isFlatGlobalAddrSpace(PtrTy.getAddressSpace()))
3308     return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3309 
3310   // If we're using MUBUF instructions for global memory, an SGPR base register
3311   // is possible. Otherwise this needs to be a VGPR.
3312   const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3313   return AMDGPU::getValueMapping(PtrBank->getID(), Size);
3314 }
3315 
3316 const RegisterBankInfo::InstructionMapping &
3317 AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
3318 
3319   const MachineFunction &MF = *MI.getParent()->getParent();
3320   const MachineRegisterInfo &MRI = MF.getRegInfo();
3321   SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3322   unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3323   Register PtrReg = MI.getOperand(1).getReg();
3324   LLT PtrTy = MRI.getType(PtrReg);
3325   unsigned AS = PtrTy.getAddressSpace();
3326   unsigned PtrSize = PtrTy.getSizeInBits();
3327 
3328   const ValueMapping *ValMapping;
3329   const ValueMapping *PtrMapping;
3330 
3331   const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3332 
3333   if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) {
3334     if (isScalarLoadLegal(MI)) {
3335       // We have a uniform instruction so we want to use an SMRD load
3336       ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3337       PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize);
3338     } else {
3339       ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3340 
3341       // If we're using MUBUF instructions for global memory, an SGPR base
3342       // register is possible. Otherwise this needs to be a VGPR.
3343       unsigned PtrBankID = Subtarget.useFlatForGlobal() ?
3344         AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID;
3345 
3346       PtrMapping = AMDGPU::getValueMapping(PtrBankID, PtrSize);
3347     }
3348   } else {
3349     ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3350     PtrMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize);
3351   }
3352 
3353   OpdsMapping[0] = ValMapping;
3354   OpdsMapping[1] = PtrMapping;
3355   const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping(
3356       1, 1, getOperandsMapping(OpdsMapping), MI.getNumOperands());
3357   return Mapping;
3358 
3359   // FIXME: Do we want to add a mapping for FLAT load, or should we just
3360   // handle that during instruction selection?
3361 }
3362 
3363 unsigned
3364 AMDGPURegisterBankInfo::getRegBankID(Register Reg,
3365                                      const MachineRegisterInfo &MRI,
3366                                      unsigned Default) const {
3367   const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
3368   return Bank ? Bank->getID() : Default;
3369 }
3370 
3371 const RegisterBankInfo::ValueMapping *
3372 AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
3373                                          const MachineRegisterInfo &MRI,
3374                                          const TargetRegisterInfo &TRI) const {
3375   // Lie and claim anything is legal, even though this needs to be an SGPR
3376   // applyMapping will have to deal with it as a waterfall loop.
3377   unsigned Bank = getRegBankID(Reg, MRI, AMDGPU::SGPRRegBankID);
3378   unsigned Size = getSizeInBits(Reg, MRI, TRI);
3379   return AMDGPU::getValueMapping(Bank, Size);
3380 }
3381 
3382 const RegisterBankInfo::ValueMapping *
3383 AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
3384                                          const MachineRegisterInfo &MRI,
3385                                          const TargetRegisterInfo &TRI) const {
3386   unsigned Size = getSizeInBits(Reg, MRI, TRI);
3387   return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3388 }
3389 
3390 const RegisterBankInfo::ValueMapping *
3391 AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
3392                                          const MachineRegisterInfo &MRI,
3393                                          const TargetRegisterInfo &TRI) const {
3394   unsigned Size = getSizeInBits(Reg, MRI, TRI);
3395   return AMDGPU::getValueMapping(AMDGPU::AGPRRegBankID, Size);
3396 }
3397 
3398 ///
3399 /// This function must return a legal mapping, because
3400 /// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
3401 /// in RegBankSelect::Mode::Fast.  Any mapping that would cause a
3402 /// VGPR to SGPR generated is illegal.
3403 ///
3404 // Operands that must be SGPRs must accept potentially divergent VGPRs as
3405 // legal. These will be dealt with in applyMappingImpl.
3406 //
3407 const RegisterBankInfo::InstructionMapping &
3408 AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
3409   const MachineFunction &MF = *MI.getParent()->getParent();
3410   const MachineRegisterInfo &MRI = MF.getRegInfo();
3411 
3412   if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) {
3413     // The default logic bothers to analyze impossible alternative mappings. We
3414     // want the most straightforward mapping, so just directly handle this.
3415     const RegisterBank *DstBank = getRegBank(MI.getOperand(0).getReg(), MRI,
3416                                              *TRI);
3417     const RegisterBank *SrcBank = getRegBank(MI.getOperand(1).getReg(), MRI,
3418                                              *TRI);
3419     assert(SrcBank && "src bank should have been assigned already");
3420     if (!DstBank)
3421       DstBank = SrcBank;
3422 
3423     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3424     if (cannotCopy(*DstBank, *SrcBank, Size))
3425       return getInvalidInstructionMapping();
3426 
3427     const ValueMapping &ValMap = getValueMapping(0, Size, *DstBank);
3428     unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2;
3429     SmallVector<const ValueMapping *, 1> OpdsMapping(OpdsMappingSize);
3430     OpdsMapping[0] = &ValMap;
3431     if (MI.getOpcode() == AMDGPU::G_FREEZE)
3432       OpdsMapping[1] = &ValMap;
3433 
3434     return getInstructionMapping(
3435         1, /*Cost*/ 1,
3436         /*OperandsMapping*/ getOperandsMapping(OpdsMapping), OpdsMappingSize);
3437   }
3438 
3439   if (MI.isRegSequence()) {
3440     // If any input is a VGPR, the result must be a VGPR. The default handling
3441     // assumes any copy between banks is legal.
3442     unsigned BankID = AMDGPU::SGPRRegBankID;
3443 
3444     for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3445       auto OpBank = getRegBankID(MI.getOperand(I).getReg(), MRI);
3446       // It doesn't make sense to use vcc or scc banks here, so just ignore
3447       // them.
3448       if (OpBank != AMDGPU::SGPRRegBankID) {
3449         BankID = AMDGPU::VGPRRegBankID;
3450         break;
3451       }
3452     }
3453     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3454 
3455     const ValueMapping &ValMap = getValueMapping(0, Size, getRegBank(BankID));
3456     return getInstructionMapping(
3457         1, /*Cost*/ 1,
3458         /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1);
3459   }
3460 
3461   // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3462   // properly.
3463   //
3464   // TODO: There are additional exec masking dependencies to analyze.
3465   if (MI.getOpcode() == TargetOpcode::G_PHI) {
3466     unsigned ResultBank = AMDGPU::InvalidRegBankID;
3467     Register DstReg = MI.getOperand(0).getReg();
3468 
3469     // Sometimes the result may have already been assigned a bank.
3470     if (const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI))
3471       ResultBank = DstBank->getID();
3472 
3473     for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3474       Register Reg = MI.getOperand(I).getReg();
3475       const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
3476 
3477       // FIXME: Assuming VGPR for any undetermined inputs.
3478       if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3479         ResultBank = AMDGPU::VGPRRegBankID;
3480         break;
3481       }
3482 
3483       // FIXME: Need to promote SGPR case to s32
3484       unsigned OpBank = Bank->getID();
3485       ResultBank = regBankBoolUnion(ResultBank, OpBank);
3486     }
3487 
3488     assert(ResultBank != AMDGPU::InvalidRegBankID);
3489 
3490     unsigned Size = MRI.getType(DstReg).getSizeInBits();
3491 
3492     const ValueMapping &ValMap =
3493         getValueMapping(0, Size, getRegBank(ResultBank));
3494     return getInstructionMapping(
3495         1, /*Cost*/ 1,
3496         /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1);
3497   }
3498 
3499   const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
3500   if (Mapping.isValid())
3501     return Mapping;
3502 
3503   SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3504 
3505   switch (MI.getOpcode()) {
3506   default:
3507     return getInvalidInstructionMapping();
3508 
3509   case AMDGPU::G_AND:
3510   case AMDGPU::G_OR:
3511   case AMDGPU::G_XOR: {
3512     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3513     if (Size == 1) {
3514       const RegisterBank *DstBank
3515         = getRegBank(MI.getOperand(0).getReg(), MRI, *TRI);
3516 
3517       unsigned TargetBankID = AMDGPU::InvalidRegBankID;
3518       unsigned BankLHS = AMDGPU::InvalidRegBankID;
3519       unsigned BankRHS = AMDGPU::InvalidRegBankID;
3520       if (DstBank) {
3521         TargetBankID = DstBank->getID();
3522         if (DstBank == &AMDGPU::VCCRegBank) {
3523           TargetBankID = AMDGPU::VCCRegBankID;
3524           BankLHS = AMDGPU::VCCRegBankID;
3525           BankRHS = AMDGPU::VCCRegBankID;
3526         } else {
3527           BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI,
3528                                  AMDGPU::SGPRRegBankID);
3529           BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
3530                                  AMDGPU::SGPRRegBankID);
3531         }
3532       } else {
3533         BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI,
3534                                AMDGPU::VCCRegBankID);
3535         BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
3536                                AMDGPU::VCCRegBankID);
3537 
3538         // Both inputs should be true booleans to produce a boolean result.
3539         if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
3540           TargetBankID = AMDGPU::VGPRRegBankID;
3541         } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
3542           TargetBankID = AMDGPU::VCCRegBankID;
3543           BankLHS = AMDGPU::VCCRegBankID;
3544           BankRHS = AMDGPU::VCCRegBankID;
3545         } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
3546           TargetBankID = AMDGPU::SGPRRegBankID;
3547         }
3548       }
3549 
3550       OpdsMapping[0] = AMDGPU::getValueMapping(TargetBankID, Size);
3551       OpdsMapping[1] = AMDGPU::getValueMapping(BankLHS, Size);
3552       OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS, Size);
3553       break;
3554     }
3555 
3556     if (Size == 64) {
3557 
3558       if (isSALUMapping(MI)) {
3559         OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size);
3560         OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
3561       } else {
3562         OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size);
3563         unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI /*, DefaultBankID*/);
3564         OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size);
3565 
3566         unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI /*, DefaultBankID*/);
3567         OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size);
3568       }
3569 
3570       break;
3571     }
3572 
3573     LLVM_FALLTHROUGH;
3574   }
3575   case AMDGPU::G_PTR_ADD:
3576   case AMDGPU::G_PTRMASK:
3577   case AMDGPU::G_ADD:
3578   case AMDGPU::G_SUB:
3579   case AMDGPU::G_MUL:
3580   case AMDGPU::G_SHL:
3581   case AMDGPU::G_LSHR:
3582   case AMDGPU::G_ASHR:
3583   case AMDGPU::G_UADDO:
3584   case AMDGPU::G_USUBO:
3585   case AMDGPU::G_UADDE:
3586   case AMDGPU::G_SADDE:
3587   case AMDGPU::G_USUBE:
3588   case AMDGPU::G_SSUBE:
3589   case AMDGPU::G_SMIN:
3590   case AMDGPU::G_SMAX:
3591   case AMDGPU::G_UMIN:
3592   case AMDGPU::G_UMAX:
3593   case AMDGPU::G_SHUFFLE_VECTOR:
3594     if (isSALUMapping(MI))
3595       return getDefaultMappingSOP(MI);
3596     LLVM_FALLTHROUGH;
3597 
3598   case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU
3599   case AMDGPU::G_SSUBSAT:
3600   case AMDGPU::G_UADDSAT:
3601   case AMDGPU::G_USUBSAT:
3602   case AMDGPU::G_FADD:
3603   case AMDGPU::G_FSUB:
3604   case AMDGPU::G_FPTOSI:
3605   case AMDGPU::G_FPTOUI:
3606   case AMDGPU::G_FMUL:
3607   case AMDGPU::G_FMA:
3608   case AMDGPU::G_FMAD:
3609   case AMDGPU::G_FSQRT:
3610   case AMDGPU::G_FFLOOR:
3611   case AMDGPU::G_FCEIL:
3612   case AMDGPU::G_FRINT:
3613   case AMDGPU::G_SITOFP:
3614   case AMDGPU::G_UITOFP:
3615   case AMDGPU::G_FPTRUNC:
3616   case AMDGPU::G_FPEXT:
3617   case AMDGPU::G_FEXP2:
3618   case AMDGPU::G_FLOG2:
3619   case AMDGPU::G_FMINNUM:
3620   case AMDGPU::G_FMAXNUM:
3621   case AMDGPU::G_FMINNUM_IEEE:
3622   case AMDGPU::G_FMAXNUM_IEEE:
3623   case AMDGPU::G_FCANONICALIZE:
3624   case AMDGPU::G_INTRINSIC_TRUNC:
3625   case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
3626   case AMDGPU::G_FSHR: // TODO: Expand for scalar
3627   case AMDGPU::G_AMDGPU_FFBH_U32:
3628   case AMDGPU::G_AMDGPU_FMIN_LEGACY:
3629   case AMDGPU::G_AMDGPU_FMAX_LEGACY:
3630   case AMDGPU::G_AMDGPU_RCP_IFLAG:
3631   case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
3632   case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
3633   case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
3634   case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
3635     return getDefaultMappingVOP(MI);
3636   case AMDGPU::G_UMULH:
3637   case AMDGPU::G_SMULH: {
3638     if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI))
3639       return getDefaultMappingSOP(MI);
3640     return getDefaultMappingVOP(MI);
3641   }
3642   case AMDGPU::G_IMPLICIT_DEF: {
3643     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3644     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3645     break;
3646   }
3647   case AMDGPU::G_FCONSTANT:
3648   case AMDGPU::G_CONSTANT:
3649   case AMDGPU::G_GLOBAL_VALUE:
3650   case AMDGPU::G_BLOCK_ADDR:
3651   case AMDGPU::G_READCYCLECOUNTER: {
3652     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3653     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3654     break;
3655   }
3656   case AMDGPU::G_FRAME_INDEX: {
3657     // TODO: This should be the same as other constants, but eliminateFrameIndex
3658     // currently assumes VALU uses.
3659     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3660     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3661     break;
3662   }
3663   case AMDGPU::G_DYN_STACKALLOC: {
3664     // Result is always uniform, and a wave reduction is needed for the source.
3665     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
3666     unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3667     OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32);
3668     break;
3669   }
3670   case AMDGPU::G_INSERT: {
3671     unsigned BankID = getMappingType(MRI, MI);
3672     unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3673     unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
3674     unsigned EltSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI);
3675     OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
3676     OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
3677     OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize);
3678     OpdsMapping[3] = nullptr;
3679     break;
3680   }
3681   case AMDGPU::G_EXTRACT: {
3682     unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3683     unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3684     unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
3685     OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
3686     OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
3687     OpdsMapping[2] = nullptr;
3688     break;
3689   }
3690   case AMDGPU::G_BUILD_VECTOR:
3691   case AMDGPU::G_BUILD_VECTOR_TRUNC: {
3692     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3693     if (DstTy == LLT::vector(2, 16)) {
3694       unsigned DstSize = DstTy.getSizeInBits();
3695       unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3696       unsigned Src0BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3697       unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
3698       unsigned DstBankID = regBankUnion(Src0BankID, Src1BankID);
3699 
3700       OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize);
3701       OpdsMapping[1] = AMDGPU::getValueMapping(Src0BankID, SrcSize);
3702       OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize);
3703       break;
3704     }
3705 
3706     LLVM_FALLTHROUGH;
3707   }
3708   case AMDGPU::G_MERGE_VALUES:
3709   case AMDGPU::G_CONCAT_VECTORS: {
3710     unsigned Bank = getMappingType(MRI, MI);
3711     unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3712     unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3713 
3714     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
3715     // Op1 and Dst should use the same register bank.
3716     for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i)
3717       OpdsMapping[i] = AMDGPU::getValueMapping(Bank, SrcSize);
3718     break;
3719   }
3720   case AMDGPU::G_BITCAST:
3721   case AMDGPU::G_INTTOPTR:
3722   case AMDGPU::G_PTRTOINT:
3723   case AMDGPU::G_BITREVERSE:
3724   case AMDGPU::G_FABS:
3725   case AMDGPU::G_FNEG: {
3726     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3727     unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3728     OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
3729     break;
3730   }
3731   case AMDGPU::G_CTLZ_ZERO_UNDEF:
3732   case AMDGPU::G_CTTZ_ZERO_UNDEF:
3733   case AMDGPU::G_CTPOP: {
3734     unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3735     unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3736     OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32);
3737 
3738     // This should really be getValueMappingSGPR64Only, but allowing the generic
3739     // code to handle the register split just makes using LegalizerHelper more
3740     // difficult.
3741     OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
3742     break;
3743   }
3744   case AMDGPU::G_TRUNC: {
3745     Register Dst = MI.getOperand(0).getReg();
3746     Register Src = MI.getOperand(1).getReg();
3747     unsigned Bank = getRegBankID(Src, MRI);
3748     unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
3749     unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
3750     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
3751     OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize);
3752     break;
3753   }
3754   case AMDGPU::G_ZEXT:
3755   case AMDGPU::G_SEXT:
3756   case AMDGPU::G_ANYEXT:
3757   case AMDGPU::G_SEXT_INREG: {
3758     Register Dst = MI.getOperand(0).getReg();
3759     Register Src = MI.getOperand(1).getReg();
3760     unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
3761     unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
3762 
3763     unsigned DstBank;
3764     const RegisterBank *SrcBank = getRegBank(Src, MRI, *TRI);
3765     assert(SrcBank);
3766     switch (SrcBank->getID()) {
3767     case AMDGPU::SGPRRegBankID:
3768       DstBank = AMDGPU::SGPRRegBankID;
3769       break;
3770     default:
3771       DstBank = AMDGPU::VGPRRegBankID;
3772       break;
3773     }
3774 
3775     // Scalar extend can use 64-bit BFE, but VGPRs require extending to
3776     // 32-bits, and then to 64.
3777     OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize);
3778     OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(),
3779                                                        SrcSize);
3780     break;
3781   }
3782   case AMDGPU::G_FCMP: {
3783     unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3784     unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI);
3785     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
3786     OpdsMapping[1] = nullptr; // Predicate Operand.
3787     OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size);
3788     OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3789     break;
3790   }
3791   case AMDGPU::G_STORE: {
3792     assert(MI.getOperand(0).isReg());
3793     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3794 
3795     // FIXME: We need to specify a different reg bank once scalar stores are
3796     // supported.
3797     const ValueMapping *ValMapping =
3798         AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3799     OpdsMapping[0] = ValMapping;
3800     OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
3801     break;
3802   }
3803   case AMDGPU::G_ICMP: {
3804     auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
3805     unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3806 
3807     // See if the result register has already been constrained to vcc, which may
3808     // happen due to control flow intrinsic lowering.
3809     unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI,
3810                                     AMDGPU::SGPRRegBankID);
3811     unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI);
3812     unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI);
3813 
3814     bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
3815                      Op2Bank == AMDGPU::SGPRRegBankID &&
3816                      Op3Bank == AMDGPU::SGPRRegBankID &&
3817       (Size == 32 || (Size == 64 &&
3818                       (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
3819                       Subtarget.hasScalarCompareEq64()));
3820 
3821     DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
3822     unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3823 
3824     // TODO: Use 32-bit for scalar output size.
3825     // SCC results will need to be copied to a 32-bit SGPR virtual register.
3826     const unsigned ResultSize = 1;
3827 
3828     OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize);
3829     OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size);
3830     OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank, Size);
3831     break;
3832   }
3833   case AMDGPU::G_EXTRACT_VECTOR_ELT: {
3834     // VGPR index can be used for waterfall when indexing a SGPR vector.
3835     unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
3836     unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3837     unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3838     unsigned IdxSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3839     unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI);
3840     unsigned OutputBankID = regBankUnion(SrcBankID, IdxBank);
3841 
3842     OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize);
3843     OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, SrcSize);
3844 
3845     // The index can be either if the source vector is VGPR.
3846     OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize);
3847     break;
3848   }
3849   case AMDGPU::G_INSERT_VECTOR_ELT: {
3850     unsigned OutputBankID = isSALUMapping(MI) ?
3851       AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3852 
3853     unsigned VecSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3854     unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3855     unsigned IdxSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits();
3856     unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
3857     unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI);
3858 
3859     OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize);
3860     OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize);
3861 
3862     // This is a weird case, because we need to break down the mapping based on
3863     // the register bank of a different operand.
3864     if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
3865       OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID,
3866                                                       InsertSize);
3867     } else {
3868       assert(InsertSize == 32 || InsertSize == 64);
3869       OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize);
3870     }
3871 
3872     // The index can be either if the source vector is VGPR.
3873     OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize);
3874     break;
3875   }
3876   case AMDGPU::G_UNMERGE_VALUES: {
3877     unsigned Bank = getMappingType(MRI, MI);
3878 
3879     // Op1 and Dst should use the same register bank.
3880     // FIXME: Shouldn't this be the default? Why do we need to handle this?
3881     for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3882       unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI);
3883       OpdsMapping[i] = AMDGPU::getValueMapping(Bank, Size);
3884     }
3885     break;
3886   }
3887   case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3888   case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3889   case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3890   case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3891   case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3892   case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3893   case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3894   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3895   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3896   case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3897   case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
3898   case AMDGPU::G_AMDGPU_BUFFER_STORE:
3899   case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3900   case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3901   case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3902   case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
3903     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3904 
3905     // rsrc
3906     OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3907 
3908     // vindex
3909     OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3910 
3911     // voffset
3912     OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3913 
3914     // soffset
3915     OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3916 
3917     // Any remaining operands are immediates and were correctly null
3918     // initialized.
3919     break;
3920   }
3921   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3922   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3923   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3924   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3925   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3926   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3927   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3928   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3929   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3930   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3931   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3932   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: {
3933     // vdata_out
3934     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3935 
3936     // vdata_in
3937     OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3938 
3939     // rsrc
3940     OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3941 
3942     // vindex
3943     OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3944 
3945     // voffset
3946     OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3947 
3948     // soffset
3949     OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
3950 
3951     // Any remaining operands are immediates and were correctly null
3952     // initialized.
3953     break;
3954   }
3955   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: {
3956     // vdata_in
3957     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3958 
3959     // rsrc
3960     OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3961 
3962     // vindex
3963     OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3964 
3965     // voffset
3966     OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3967 
3968     // soffset
3969     OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3970     break;
3971   }
3972   case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3973     // vdata_out
3974     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3975 
3976     // vdata_in
3977     OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3978 
3979     // cmp
3980     OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3981 
3982     // rsrc
3983     OpdsMapping[3] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3984 
3985     // vindex
3986     OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3987 
3988     // voffset
3989     OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
3990 
3991     // soffset
3992     OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI);
3993 
3994     // Any remaining operands are immediates and were correctly null
3995     // initialized.
3996     break;
3997   }
3998   case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: {
3999     // Lie and claim everything is legal, even though some need to be
4000     // SGPRs. applyMapping will have to deal with it as a waterfall loop.
4001     OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4002     OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4003 
4004     // We need to convert this to a MUBUF if either the resource of offset is
4005     // VGPR.
4006     unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4007     unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4008     unsigned ResultBank = regBankUnion(RSrcBank, OffsetBank);
4009 
4010     unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4011     OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0);
4012     break;
4013   }
4014   case AMDGPU::G_INTRINSIC: {
4015     switch (MI.getIntrinsicID()) {
4016     default:
4017       return getInvalidInstructionMapping();
4018     case Intrinsic::amdgcn_div_fmas:
4019     case Intrinsic::amdgcn_div_fixup:
4020     case Intrinsic::amdgcn_trig_preop:
4021     case Intrinsic::amdgcn_sin:
4022     case Intrinsic::amdgcn_cos:
4023     case Intrinsic::amdgcn_log_clamp:
4024     case Intrinsic::amdgcn_rcp:
4025     case Intrinsic::amdgcn_rcp_legacy:
4026     case Intrinsic::amdgcn_sqrt:
4027     case Intrinsic::amdgcn_rsq:
4028     case Intrinsic::amdgcn_rsq_legacy:
4029     case Intrinsic::amdgcn_rsq_clamp:
4030     case Intrinsic::amdgcn_fmul_legacy:
4031     case Intrinsic::amdgcn_ldexp:
4032     case Intrinsic::amdgcn_frexp_mant:
4033     case Intrinsic::amdgcn_frexp_exp:
4034     case Intrinsic::amdgcn_fract:
4035     case Intrinsic::amdgcn_cvt_pkrtz:
4036     case Intrinsic::amdgcn_cvt_pknorm_i16:
4037     case Intrinsic::amdgcn_cvt_pknorm_u16:
4038     case Intrinsic::amdgcn_cvt_pk_i16:
4039     case Intrinsic::amdgcn_cvt_pk_u16:
4040     case Intrinsic::amdgcn_fmed3:
4041     case Intrinsic::amdgcn_cubeid:
4042     case Intrinsic::amdgcn_cubema:
4043     case Intrinsic::amdgcn_cubesc:
4044     case Intrinsic::amdgcn_cubetc:
4045     case Intrinsic::amdgcn_sffbh:
4046     case Intrinsic::amdgcn_fmad_ftz:
4047     case Intrinsic::amdgcn_mbcnt_lo:
4048     case Intrinsic::amdgcn_mbcnt_hi:
4049     case Intrinsic::amdgcn_mul_u24:
4050     case Intrinsic::amdgcn_mul_i24:
4051     case Intrinsic::amdgcn_lerp:
4052     case Intrinsic::amdgcn_sad_u8:
4053     case Intrinsic::amdgcn_msad_u8:
4054     case Intrinsic::amdgcn_sad_hi_u8:
4055     case Intrinsic::amdgcn_sad_u16:
4056     case Intrinsic::amdgcn_qsad_pk_u16_u8:
4057     case Intrinsic::amdgcn_mqsad_pk_u16_u8:
4058     case Intrinsic::amdgcn_mqsad_u32_u8:
4059     case Intrinsic::amdgcn_cvt_pk_u8_f32:
4060     case Intrinsic::amdgcn_alignbit:
4061     case Intrinsic::amdgcn_alignbyte:
4062     case Intrinsic::amdgcn_fdot2:
4063     case Intrinsic::amdgcn_sdot2:
4064     case Intrinsic::amdgcn_udot2:
4065     case Intrinsic::amdgcn_sdot4:
4066     case Intrinsic::amdgcn_udot4:
4067     case Intrinsic::amdgcn_sdot8:
4068     case Intrinsic::amdgcn_udot8:
4069       return getDefaultMappingVOP(MI);
4070     case Intrinsic::amdgcn_sbfe:
4071     case Intrinsic::amdgcn_ubfe:
4072       if (isSALUMapping(MI))
4073         return getDefaultMappingSOP(MI);
4074       return getDefaultMappingVOP(MI);
4075     case Intrinsic::amdgcn_ds_swizzle:
4076     case Intrinsic::amdgcn_ds_permute:
4077     case Intrinsic::amdgcn_ds_bpermute:
4078     case Intrinsic::amdgcn_update_dpp:
4079     case Intrinsic::amdgcn_mov_dpp8:
4080     case Intrinsic::amdgcn_mov_dpp:
4081     case Intrinsic::amdgcn_wwm:
4082     case Intrinsic::amdgcn_wqm:
4083     case Intrinsic::amdgcn_softwqm:
4084     case Intrinsic::amdgcn_set_inactive:
4085       return getDefaultMappingAllVGPR(MI);
4086     case Intrinsic::amdgcn_kernarg_segment_ptr:
4087     case Intrinsic::amdgcn_s_getpc:
4088     case Intrinsic::amdgcn_groupstaticsize:
4089     case Intrinsic::amdgcn_reloc_constant:
4090     case Intrinsic::returnaddress: {
4091       unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4092       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4093       break;
4094     }
4095     case Intrinsic::amdgcn_wqm_vote: {
4096       unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4097       OpdsMapping[0] = OpdsMapping[2]
4098         = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size);
4099       break;
4100     }
4101     case Intrinsic::amdgcn_ps_live: {
4102       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4103       break;
4104     }
4105     case Intrinsic::amdgcn_div_scale: {
4106       unsigned Dst0Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4107       unsigned Dst1Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
4108       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Dst0Size);
4109       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size);
4110 
4111       unsigned SrcSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits();
4112       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4113       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4114       break;
4115     }
4116     case Intrinsic::amdgcn_class: {
4117       Register Src0Reg = MI.getOperand(2).getReg();
4118       Register Src1Reg = MI.getOperand(3).getReg();
4119       unsigned Src0Size = MRI.getType(Src0Reg).getSizeInBits();
4120       unsigned Src1Size = MRI.getType(Src1Reg).getSizeInBits();
4121       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4122       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize);
4123       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size);
4124       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size);
4125       break;
4126     }
4127     case Intrinsic::amdgcn_icmp:
4128     case Intrinsic::amdgcn_fcmp: {
4129       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4130       // This is not VCCRegBank because this is not used in boolean contexts.
4131       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4132       unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4133       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
4134       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
4135       break;
4136     }
4137     case Intrinsic::amdgcn_readlane: {
4138       // This must be an SGPR, but accept a VGPR.
4139       Register IdxReg = MI.getOperand(3).getReg();
4140       unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
4141       unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID);
4142       OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4143       LLVM_FALLTHROUGH;
4144     }
4145     case Intrinsic::amdgcn_readfirstlane: {
4146       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4147       unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4148       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4149       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4150       break;
4151     }
4152     case Intrinsic::amdgcn_writelane: {
4153       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4154       Register SrcReg = MI.getOperand(2).getReg();
4155       unsigned SrcSize = MRI.getType(SrcReg).getSizeInBits();
4156       unsigned SrcBank = getRegBankID(SrcReg, MRI, AMDGPU::SGPRRegBankID);
4157       Register IdxReg = MI.getOperand(3).getReg();
4158       unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
4159       unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID);
4160       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4161 
4162       // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
4163       // to legalize.
4164       OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize);
4165       OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4166       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4167       break;
4168     }
4169     case Intrinsic::amdgcn_if_break: {
4170       unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4171       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4172       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4173       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4174       break;
4175     }
4176     case Intrinsic::amdgcn_permlane16:
4177     case Intrinsic::amdgcn_permlanex16: {
4178       unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4179       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4180       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4181       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4182       OpdsMapping[4] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4183       OpdsMapping[5] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4184       break;
4185     }
4186     case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
4187     case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
4188     case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
4189     case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
4190     case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
4191     case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
4192     case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
4193     case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
4194     case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
4195     case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
4196     case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
4197     case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
4198     case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
4199     case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
4200     case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
4201     case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
4202     case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
4203     case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
4204     case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
4205     case Intrinsic::amdgcn_mfma_f32_32x32x4bf16: {
4206       // Default for MAI intrinsics.
4207       // srcC can also be an immediate which can be folded later.
4208       // FIXME: Should we eventually add an alternative mapping with AGPR src
4209       // for srcA/srcB?
4210       //
4211       // vdst, srcA, srcB, srcC
4212       OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4213       OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4214       OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4215       OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4216       break;
4217     }
4218     case Intrinsic::amdgcn_interp_p1:
4219     case Intrinsic::amdgcn_interp_p2:
4220     case Intrinsic::amdgcn_interp_mov:
4221     case Intrinsic::amdgcn_interp_p1_f16:
4222     case Intrinsic::amdgcn_interp_p2_f16: {
4223       const int M0Idx = MI.getNumOperands() - 1;
4224       Register M0Reg = MI.getOperand(M0Idx).getReg();
4225       unsigned M0Bank = getRegBankID(M0Reg, MRI, AMDGPU::SGPRRegBankID);
4226       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4227 
4228       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4229       for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I)
4230         OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4231 
4232       // Must be SGPR, but we must take whatever the original bank is and fix it
4233       // later.
4234       OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32);
4235       break;
4236     }
4237     case Intrinsic::amdgcn_ballot: {
4238       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4239       unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4240       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4241       OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize);
4242       break;
4243     }
4244     }
4245     break;
4246   }
4247   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4248   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
4249     auto IntrID = MI.getIntrinsicID();
4250     const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(IntrID);
4251     assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic");
4252     // Non-images can have complications from operands that allow both SGPR
4253     // and VGPR. For now it's too complicated to figure out the final opcode
4254     // to derive the register bank from the MCInstrDesc.
4255     assert(RSrcIntrin->IsImage);
4256     return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg);
4257   }
4258   case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
4259     auto IntrID = MI.getIntrinsicID();
4260     switch (IntrID) {
4261     case Intrinsic::amdgcn_s_getreg:
4262     case Intrinsic::amdgcn_s_memtime:
4263     case Intrinsic::amdgcn_s_memrealtime:
4264     case Intrinsic::amdgcn_s_get_waveid_in_workgroup: {
4265       unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4266       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4267       break;
4268     }
4269     case Intrinsic::amdgcn_global_atomic_fadd:
4270     case Intrinsic::amdgcn_global_atomic_csub:
4271       return getDefaultMappingAllVGPR(MI);
4272     case Intrinsic::amdgcn_ds_ordered_add:
4273     case Intrinsic::amdgcn_ds_ordered_swap: {
4274       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4275       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4276       unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4277                                  AMDGPU::SGPRRegBankID);
4278       OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32);
4279       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4280       break;
4281     }
4282     case Intrinsic::amdgcn_ds_append:
4283     case Intrinsic::amdgcn_ds_consume: {
4284       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4285       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4286       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4287       break;
4288     }
4289     case Intrinsic::amdgcn_exp_compr:
4290       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4291       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4292       break;
4293     case Intrinsic::amdgcn_exp:
4294       // FIXME: Could we support packed types here?
4295       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4296       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4297       OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4298       OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4299       break;
4300     case Intrinsic::amdgcn_s_sendmsg:
4301     case Intrinsic::amdgcn_s_sendmsghalt: {
4302       // This must be an SGPR, but accept a VGPR.
4303       unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4304                                    AMDGPU::SGPRRegBankID);
4305       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4306       break;
4307     }
4308     case Intrinsic::amdgcn_s_setreg: {
4309       // This must be an SGPR, but accept a VGPR.
4310       unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4311                                    AMDGPU::SGPRRegBankID);
4312       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4313       break;
4314     }
4315     case Intrinsic::amdgcn_end_cf: {
4316       unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4317       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4318       break;
4319     }
4320     case Intrinsic::amdgcn_else: {
4321       unsigned WaveSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4322       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4323       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
4324       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
4325       break;
4326     }
4327     case Intrinsic::amdgcn_kill: {
4328       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4329       break;
4330     }
4331     case Intrinsic::amdgcn_raw_buffer_load:
4332     case Intrinsic::amdgcn_raw_tbuffer_load: {
4333       // FIXME: Should make intrinsic ID the last operand of the instruction,
4334       // then this would be the same as store
4335       OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4336       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4337       OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4338       OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4339       break;
4340     }
4341     case Intrinsic::amdgcn_raw_buffer_store:
4342     case Intrinsic::amdgcn_raw_buffer_store_format:
4343     case Intrinsic::amdgcn_raw_tbuffer_store: {
4344       OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4345       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4346       OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4347       OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4348       break;
4349     }
4350     case Intrinsic::amdgcn_struct_buffer_load:
4351     case Intrinsic::amdgcn_struct_tbuffer_load: {
4352       OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4353       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4354       OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4355       OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4356       OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4357       break;
4358     }
4359     case Intrinsic::amdgcn_struct_buffer_store:
4360     case Intrinsic::amdgcn_struct_tbuffer_store: {
4361       OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4362       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4363       OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4364       OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4365       OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4366       break;
4367     }
4368     case Intrinsic::amdgcn_init_exec_from_input: {
4369       unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4370       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4371       break;
4372     }
4373     case Intrinsic::amdgcn_ds_gws_init:
4374     case Intrinsic::amdgcn_ds_gws_barrier:
4375     case Intrinsic::amdgcn_ds_gws_sema_br: {
4376       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4377 
4378       // This must be an SGPR, but accept a VGPR.
4379       unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4380                                    AMDGPU::SGPRRegBankID);
4381       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4382       break;
4383     }
4384     case Intrinsic::amdgcn_ds_gws_sema_v:
4385     case Intrinsic::amdgcn_ds_gws_sema_p:
4386     case Intrinsic::amdgcn_ds_gws_sema_release_all: {
4387       // This must be an SGPR, but accept a VGPR.
4388       unsigned Bank = getRegBankID(MI.getOperand(1).getReg(), MRI,
4389                                    AMDGPU::SGPRRegBankID);
4390       OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32);
4391       break;
4392     }
4393     default:
4394       return getInvalidInstructionMapping();
4395     }
4396     break;
4397   }
4398   case AMDGPU::G_SELECT: {
4399     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4400     unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4401                                     AMDGPU::SGPRRegBankID);
4402     unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI,
4403                                     AMDGPU::SGPRRegBankID);
4404     bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
4405                     Op3Bank == AMDGPU::SGPRRegBankID;
4406 
4407     unsigned CondBankDefault = SGPRSrcs ?
4408       AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4409     unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI,
4410                                      CondBankDefault);
4411     if (CondBank == AMDGPU::SGPRRegBankID)
4412       CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4413     else if (CondBank == AMDGPU::VGPRRegBankID)
4414       CondBank = AMDGPU::VCCRegBankID;
4415 
4416     unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
4417       AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4418 
4419     assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
4420 
4421     // TODO: Should report 32-bit for scalar condition type.
4422     if (Size == 64) {
4423       OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4424       OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
4425       OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4426       OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4427     } else {
4428       OpdsMapping[0] = AMDGPU::getValueMapping(Bank, Size);
4429       OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
4430       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size);
4431       OpdsMapping[3] = AMDGPU::getValueMapping(Bank, Size);
4432     }
4433 
4434     break;
4435   }
4436 
4437   case AMDGPU::G_LOAD:
4438   case AMDGPU::G_ZEXTLOAD:
4439   case AMDGPU::G_SEXTLOAD:
4440     return getInstrMappingForLoad(MI);
4441 
4442   case AMDGPU::G_ATOMICRMW_XCHG:
4443   case AMDGPU::G_ATOMICRMW_ADD:
4444   case AMDGPU::G_ATOMICRMW_SUB:
4445   case AMDGPU::G_ATOMICRMW_AND:
4446   case AMDGPU::G_ATOMICRMW_OR:
4447   case AMDGPU::G_ATOMICRMW_XOR:
4448   case AMDGPU::G_ATOMICRMW_MAX:
4449   case AMDGPU::G_ATOMICRMW_MIN:
4450   case AMDGPU::G_ATOMICRMW_UMAX:
4451   case AMDGPU::G_ATOMICRMW_UMIN:
4452   case AMDGPU::G_ATOMICRMW_FADD:
4453   case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
4454   case AMDGPU::G_AMDGPU_ATOMIC_INC:
4455   case AMDGPU::G_AMDGPU_ATOMIC_DEC:
4456   case AMDGPU::G_AMDGPU_ATOMIC_FMIN:
4457   case AMDGPU::G_AMDGPU_ATOMIC_FMAX: {
4458     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4459     OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
4460     OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4461     break;
4462   }
4463   case AMDGPU::G_ATOMIC_CMPXCHG: {
4464     OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4465     OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
4466     OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4467     OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4468     break;
4469   }
4470   case AMDGPU::G_BRCOND: {
4471     unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI,
4472                                  AMDGPU::SGPRRegBankID);
4473     assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
4474     if (Bank != AMDGPU::SGPRRegBankID)
4475       Bank = AMDGPU::VCCRegBankID;
4476 
4477     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
4478     break;
4479   }
4480   }
4481 
4482   return getInstructionMapping(/*ID*/1, /*Cost*/1,
4483                                getOperandsMapping(OpdsMapping),
4484                                MI.getNumOperands());
4485 }
4486