1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPUArgumentUsageInfo.h"
16 #include "AMDGPUISelLowering.h" // For AMDGPUISD
17 #include "AMDGPUInstrInfo.h"
18 #include "AMDGPUPerfHintAnalysis.h"
19 #include "AMDGPUSubtarget.h"
20 #include "AMDGPUTargetMachine.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "SIDefines.h"
23 #include "SIISelLowering.h"
24 #include "SIInstrInfo.h"
25 #include "SIMachineFunctionInfo.h"
26 #include "SIRegisterInfo.h"
27 #include "llvm/ADT/APInt.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
31 #include "llvm/Analysis/LoopInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/FunctionLoweringInfo.h"
34 #include "llvm/CodeGen/ISDOpcodes.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/SelectionDAG.h"
38 #include "llvm/CodeGen/SelectionDAGISel.h"
39 #include "llvm/CodeGen/SelectionDAGNodes.h"
40 #include "llvm/CodeGen/ValueTypes.h"
41 #include "llvm/IR/BasicBlock.h"
42 #include "llvm/InitializePasses.h"
43 #ifdef EXPENSIVE_CHECKS
44 #include "llvm/IR/Dominators.h"
45 #endif
46 #include "llvm/IR/Instruction.h"
47 #include "llvm/MC/MCInstrDesc.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CodeGen.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/MachineValueType.h"
52 #include "llvm/Support/MathExtras.h"
53 #include <cassert>
54 #include <cstdint>
55 #include <new>
56 #include <vector>
57 
58 #define DEBUG_TYPE "isel"
59 
60 using namespace llvm;
61 
62 namespace llvm {
63 
64 class R600InstrInfo;
65 
66 } // end namespace llvm
67 
68 //===----------------------------------------------------------------------===//
69 // Instruction Selector Implementation
70 //===----------------------------------------------------------------------===//
71 
72 namespace {
73 
74 static bool isNullConstantOrUndef(SDValue V) {
75   if (V.isUndef())
76     return true;
77 
78   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
79   return Const != nullptr && Const->isNullValue();
80 }
81 
82 static bool getConstantValue(SDValue N, uint32_t &Out) {
83   // This is only used for packed vectors, where ussing 0 for undef should
84   // always be good.
85   if (N.isUndef()) {
86     Out = 0;
87     return true;
88   }
89 
90   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
91     Out = C->getAPIntValue().getSExtValue();
92     return true;
93   }
94 
95   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
96     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
97     return true;
98   }
99 
100   return false;
101 }
102 
103 // TODO: Handle undef as zero
104 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
105                                  bool Negate = false) {
106   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
107   uint32_t LHSVal, RHSVal;
108   if (getConstantValue(N->getOperand(0), LHSVal) &&
109       getConstantValue(N->getOperand(1), RHSVal)) {
110     SDLoc SL(N);
111     uint32_t K = Negate ?
112       (-LHSVal & 0xffff) | (-RHSVal << 16) :
113       (LHSVal & 0xffff) | (RHSVal << 16);
114     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
115                               DAG.getTargetConstant(K, SL, MVT::i32));
116   }
117 
118   return nullptr;
119 }
120 
121 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
122   return packConstantV2I16(N, DAG, true);
123 }
124 
125 /// AMDGPU specific code to select AMDGPU machine instructions for
126 /// SelectionDAG operations.
127 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
128   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
129   // make the right decision when generating code for different targets.
130   const GCNSubtarget *Subtarget;
131 
132   // Default FP mode for the current function.
133   AMDGPU::SIModeRegisterDefaults Mode;
134 
135   bool EnableLateStructurizeCFG;
136 
137 public:
138   explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
139                               CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
140     : SelectionDAGISel(*TM, OptLevel) {
141     EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
142   }
143   ~AMDGPUDAGToDAGISel() override = default;
144 
145   void getAnalysisUsage(AnalysisUsage &AU) const override {
146     AU.addRequired<AMDGPUArgumentUsageInfo>();
147     AU.addRequired<LegacyDivergenceAnalysis>();
148 #ifdef EXPENSIVE_CHECKS
149     AU.addRequired<DominatorTreeWrapperPass>();
150     AU.addRequired<LoopInfoWrapperPass>();
151 #endif
152     SelectionDAGISel::getAnalysisUsage(AU);
153   }
154 
155   bool matchLoadD16FromBuildVector(SDNode *N) const;
156 
157   bool runOnMachineFunction(MachineFunction &MF) override;
158   void PreprocessISelDAG() override;
159   void Select(SDNode *N) override;
160   StringRef getPassName() const override;
161   void PostprocessISelDAG() override;
162 
163 protected:
164   void SelectBuildVector(SDNode *N, unsigned RegClassID);
165 
166 private:
167   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
168   bool isNoNanSrc(SDValue N) const;
169   bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
170   bool isNegInlineImmediate(const SDNode *N) const {
171     return isInlineImmediate(N, true);
172   }
173 
174   bool isInlineImmediate16(int64_t Imm) const {
175     return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());
176   }
177 
178   bool isInlineImmediate32(int64_t Imm) const {
179     return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());
180   }
181 
182   bool isInlineImmediate64(int64_t Imm) const {
183     return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());
184   }
185 
186   bool isInlineImmediate(const APFloat &Imm) const {
187     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
188   }
189 
190   bool isVGPRImm(const SDNode *N) const;
191   bool isUniformLoad(const SDNode *N) const;
192   bool isUniformBr(const SDNode *N) const;
193 
194   bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
195                                   SDValue &RHS) const;
196 
197   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
198 
199   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
200   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
201   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
202 
203   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
204   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
205   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
206   bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
207   bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
208                         unsigned Size) const;
209   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
210   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
211                                  SDValue &Offset1) const;
212   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
213                                   SDValue &Offset1) const;
214   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
215                           SDValue &Offset1, unsigned Size) const;
216   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
217                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
218                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
219                    SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
220   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
221                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
222                          SDValue &SLC, SDValue &TFE, SDValue &DLC,
223                          SDValue &SWZ) const;
224   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
225                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
226                          SDValue &SLC) const;
227   bool SelectMUBUFScratchOffen(SDNode *Parent,
228                                SDValue Addr, SDValue &RSrc, SDValue &VAddr,
229                                SDValue &SOffset, SDValue &ImmOffset) const;
230   bool SelectMUBUFScratchOffset(SDNode *Parent,
231                                 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
232                                 SDValue &Offset) const;
233 
234   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
235                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
236                          SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
237   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
238                          SDValue &Offset, SDValue &SLC) const;
239   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
240                          SDValue &Offset) const;
241 
242   template <bool IsSigned>
243   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
244                         SDValue &Offset) const;
245   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
246                          SDValue &VOffset, SDValue &Offset) const;
247   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
248                           SDValue &Offset) const;
249 
250   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
251                         bool &Imm) const;
252   SDValue Expand32BitAddress(SDValue Addr) const;
253   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
254                   bool &Imm) const;
255   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
256   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
257   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
258   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
259   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
260   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
261 
262   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
263   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
264                           bool AllowAbs = true) const;
265   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
266   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
267   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
268   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
269                        SDValue &Clamp, SDValue &Omod) const;
270   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
271                         SDValue &Clamp, SDValue &Omod) const;
272   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
273                          SDValue &Clamp, SDValue &Omod) const;
274 
275   bool SelectVOP3OMods(SDValue In, SDValue &Src,
276                        SDValue &Clamp, SDValue &Omod) const;
277 
278   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
279 
280   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
281 
282   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
283   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
284   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
285 
286   SDValue getHi16Elt(SDValue In) const;
287 
288   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
289 
290   void SelectADD_SUB_I64(SDNode *N);
291   void SelectAddcSubb(SDNode *N);
292   void SelectUADDO_USUBO(SDNode *N);
293   void SelectDIV_SCALE(SDNode *N);
294   void SelectMAD_64_32(SDNode *N);
295   void SelectFMA_W_CHAIN(SDNode *N);
296   void SelectFMUL_W_CHAIN(SDNode *N);
297 
298   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
299                    uint32_t Offset, uint32_t Width);
300   void SelectS_BFEFromShifts(SDNode *N);
301   void SelectS_BFE(SDNode *N);
302   bool isCBranchSCC(const SDNode *N) const;
303   void SelectBRCOND(SDNode *N);
304   void SelectFMAD_FMA(SDNode *N);
305   void SelectATOMIC_CMP_SWAP(SDNode *N);
306   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
307   void SelectDS_GWS(SDNode *N, unsigned IntrID);
308   void SelectInterpP1F16(SDNode *N);
309   void SelectINTRINSIC_W_CHAIN(SDNode *N);
310   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
311   void SelectINTRINSIC_VOID(SDNode *N);
312 
313 protected:
314   // Include the pieces autogenerated from the target description.
315 #include "AMDGPUGenDAGISel.inc"
316 };
317 
318 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
319   const R600Subtarget *Subtarget;
320 
321   bool isConstantLoad(const MemSDNode *N, int cbID) const;
322   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
323   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
324                                        SDValue& Offset);
325 public:
326   explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
327       AMDGPUDAGToDAGISel(TM, OptLevel) {}
328 
329   void Select(SDNode *N) override;
330 
331   bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
332                           SDValue &Offset) override;
333   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
334                           SDValue &Offset) override;
335 
336   bool runOnMachineFunction(MachineFunction &MF) override;
337 
338   void PreprocessISelDAG() override {}
339 
340 protected:
341   // Include the pieces autogenerated from the target description.
342 #include "R600GenDAGISel.inc"
343 };
344 
345 static SDValue stripBitcast(SDValue Val) {
346   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
347 }
348 
349 // Figure out if this is really an extract of the high 16-bits of a dword.
350 static bool isExtractHiElt(SDValue In, SDValue &Out) {
351   In = stripBitcast(In);
352   if (In.getOpcode() != ISD::TRUNCATE)
353     return false;
354 
355   SDValue Srl = In.getOperand(0);
356   if (Srl.getOpcode() == ISD::SRL) {
357     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
358       if (ShiftAmt->getZExtValue() == 16) {
359         Out = stripBitcast(Srl.getOperand(0));
360         return true;
361       }
362     }
363   }
364 
365   return false;
366 }
367 
368 // Look through operations that obscure just looking at the low 16-bits of the
369 // same register.
370 static SDValue stripExtractLoElt(SDValue In) {
371   if (In.getOpcode() == ISD::TRUNCATE) {
372     SDValue Src = In.getOperand(0);
373     if (Src.getValueType().getSizeInBits() == 32)
374       return stripBitcast(Src);
375   }
376 
377   return In;
378 }
379 
380 }  // end anonymous namespace
381 
382 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
383                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
384 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
385 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
386 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
387 #ifdef EXPENSIVE_CHECKS
388 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
389 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
390 #endif
391 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
392                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
393 
394 /// This pass converts a legalized DAG into a AMDGPU-specific
395 // DAG, ready for instruction scheduling.
396 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
397                                         CodeGenOpt::Level OptLevel) {
398   return new AMDGPUDAGToDAGISel(TM, OptLevel);
399 }
400 
401 /// This pass converts a legalized DAG into a R600-specific
402 // DAG, ready for instruction scheduling.
403 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
404                                       CodeGenOpt::Level OptLevel) {
405   return new R600DAGToDAGISel(TM, OptLevel);
406 }
407 
408 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
409 #ifdef EXPENSIVE_CHECKS
410   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
411   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
412   for (auto &L : LI->getLoopsInPreorder()) {
413     assert(L->isLCSSAForm(DT));
414   }
415 #endif
416   Subtarget = &MF.getSubtarget<GCNSubtarget>();
417   Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
418   return SelectionDAGISel::runOnMachineFunction(MF);
419 }
420 
421 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
422   assert(Subtarget->d16PreservesUnusedBits());
423   MVT VT = N->getValueType(0).getSimpleVT();
424   if (VT != MVT::v2i16 && VT != MVT::v2f16)
425     return false;
426 
427   SDValue Lo = N->getOperand(0);
428   SDValue Hi = N->getOperand(1);
429 
430   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
431 
432   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
433   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
434   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
435 
436   // Need to check for possible indirect dependencies on the other half of the
437   // vector to avoid introducing a cycle.
438   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
439     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
440 
441     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
442     SDValue Ops[] = {
443       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
444     };
445 
446     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
447     if (LdHi->getMemoryVT() == MVT::i8) {
448       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
449         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
450     } else {
451       assert(LdHi->getMemoryVT() == MVT::i16);
452     }
453 
454     SDValue NewLoadHi =
455       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
456                                   Ops, LdHi->getMemoryVT(),
457                                   LdHi->getMemOperand());
458 
459     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
460     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
461     return true;
462   }
463 
464   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
465   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
466   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
467   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
468   if (LdLo && Lo.hasOneUse()) {
469     SDValue TiedIn = getHi16Elt(Hi);
470     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
471       return false;
472 
473     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
474     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
475     if (LdLo->getMemoryVT() == MVT::i8) {
476       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
477         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
478     } else {
479       assert(LdLo->getMemoryVT() == MVT::i16);
480     }
481 
482     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
483 
484     SDValue Ops[] = {
485       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
486     };
487 
488     SDValue NewLoadLo =
489       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
490                                   Ops, LdLo->getMemoryVT(),
491                                   LdLo->getMemOperand());
492 
493     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
494     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
495     return true;
496   }
497 
498   return false;
499 }
500 
501 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
502   if (!Subtarget->d16PreservesUnusedBits())
503     return;
504 
505   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
506 
507   bool MadeChange = false;
508   while (Position != CurDAG->allnodes_begin()) {
509     SDNode *N = &*--Position;
510     if (N->use_empty())
511       continue;
512 
513     switch (N->getOpcode()) {
514     case ISD::BUILD_VECTOR:
515       MadeChange |= matchLoadD16FromBuildVector(N);
516       break;
517     default:
518       break;
519     }
520   }
521 
522   if (MadeChange) {
523     CurDAG->RemoveDeadNodes();
524     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
525                CurDAG->dump(););
526   }
527 }
528 
529 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
530   if (TM.Options.NoNaNsFPMath)
531     return true;
532 
533   // TODO: Move into isKnownNeverNaN
534   if (N->getFlags().hasNoNaNs())
535     return true;
536 
537   return CurDAG->isKnownNeverNaN(N);
538 }
539 
540 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
541                                            bool Negated) const {
542   if (N->isUndef())
543     return true;
544 
545   const SIInstrInfo *TII = Subtarget->getInstrInfo();
546   if (Negated) {
547     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
548       return TII->isInlineConstant(-C->getAPIntValue());
549 
550     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
551       return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
552 
553   } else {
554     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
555       return TII->isInlineConstant(C->getAPIntValue());
556 
557     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
558       return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
559   }
560 
561   return false;
562 }
563 
564 /// Determine the register class for \p OpNo
565 /// \returns The register class of the virtual register that will be used for
566 /// the given operand number \OpNo or NULL if the register class cannot be
567 /// determined.
568 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
569                                                           unsigned OpNo) const {
570   if (!N->isMachineOpcode()) {
571     if (N->getOpcode() == ISD::CopyToReg) {
572       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
573       if (Reg.isVirtual()) {
574         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
575         return MRI.getRegClass(Reg);
576       }
577 
578       const SIRegisterInfo *TRI
579         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
580       return TRI->getPhysRegClass(Reg);
581     }
582 
583     return nullptr;
584   }
585 
586   switch (N->getMachineOpcode()) {
587   default: {
588     const MCInstrDesc &Desc =
589         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
590     unsigned OpIdx = Desc.getNumDefs() + OpNo;
591     if (OpIdx >= Desc.getNumOperands())
592       return nullptr;
593     int RegClass = Desc.OpInfo[OpIdx].RegClass;
594     if (RegClass == -1)
595       return nullptr;
596 
597     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
598   }
599   case AMDGPU::REG_SEQUENCE: {
600     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
601     const TargetRegisterClass *SuperRC =
602         Subtarget->getRegisterInfo()->getRegClass(RCID);
603 
604     SDValue SubRegOp = N->getOperand(OpNo + 1);
605     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
606     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
607                                                               SubRegIdx);
608   }
609   }
610 }
611 
612 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
613                                          SDValue Glue) const {
614   SmallVector <SDValue, 8> Ops;
615   Ops.push_back(NewChain); // Replace the chain.
616   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
617     Ops.push_back(N->getOperand(i));
618 
619   Ops.push_back(Glue);
620   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
621 }
622 
623 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
624   const SITargetLowering& Lowering =
625     *static_cast<const SITargetLowering*>(getTargetLowering());
626 
627   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
628 
629   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
630   return glueCopyToOp(N, M0, M0.getValue(1));
631 }
632 
633 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
634   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
635   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
636     if (Subtarget->ldsRequiresM0Init())
637       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
638   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
639     MachineFunction &MF = CurDAG->getMachineFunction();
640     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
641     return
642         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
643   }
644   return N;
645 }
646 
647 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
648                                                   EVT VT) const {
649   SDNode *Lo = CurDAG->getMachineNode(
650       AMDGPU::S_MOV_B32, DL, MVT::i32,
651       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
652   SDNode *Hi =
653       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
654                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
655   const SDValue Ops[] = {
656       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
657       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
658       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
659 
660   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
661 }
662 
663 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
664   EVT VT = N->getValueType(0);
665   unsigned NumVectorElts = VT.getVectorNumElements();
666   EVT EltVT = VT.getVectorElementType();
667   SDLoc DL(N);
668   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
669 
670   if (NumVectorElts == 1) {
671     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
672                          RegClass);
673     return;
674   }
675 
676   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
677                                   "supported yet");
678   // 32 = Max Num Vector Elements
679   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
680   // 1 = Vector Register Class
681   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
682 
683   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
684                Triple::amdgcn;
685   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
686   bool IsRegSeq = true;
687   unsigned NOps = N->getNumOperands();
688   for (unsigned i = 0; i < NOps; i++) {
689     // XXX: Why is this here?
690     if (isa<RegisterSDNode>(N->getOperand(i))) {
691       IsRegSeq = false;
692       break;
693     }
694     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
695                          : R600RegisterInfo::getSubRegFromChannel(i);
696     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
697     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
698   }
699   if (NOps != NumVectorElts) {
700     // Fill in the missing undef elements if this was a scalar_to_vector.
701     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
702     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
703                                                    DL, EltVT);
704     for (unsigned i = NOps; i < NumVectorElts; ++i) {
705       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
706                            : R600RegisterInfo::getSubRegFromChannel(i);
707       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
708       RegSeqArgs[1 + (2 * i) + 1] =
709           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
710     }
711   }
712 
713   if (!IsRegSeq)
714     SelectCode(N);
715   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
716 }
717 
718 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
719   unsigned int Opc = N->getOpcode();
720   if (N->isMachineOpcode()) {
721     N->setNodeId(-1);
722     return;   // Already selected.
723   }
724 
725   // isa<MemSDNode> almost works but is slightly too permissive for some DS
726   // intrinsics.
727   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
728       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
729        Opc == ISD::ATOMIC_LOAD_FADD ||
730        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
731        Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
732     N = glueCopyToM0LDSInit(N);
733     SelectCode(N);
734     return;
735   }
736 
737   switch (Opc) {
738   default:
739     break;
740   // We are selecting i64 ADD here instead of custom lower it during
741   // DAG legalization, so we can fold some i64 ADDs used for address
742   // calculation into the LOAD and STORE instructions.
743   case ISD::ADDC:
744   case ISD::ADDE:
745   case ISD::SUBC:
746   case ISD::SUBE: {
747     if (N->getValueType(0) != MVT::i64)
748       break;
749 
750     SelectADD_SUB_I64(N);
751     return;
752   }
753   case ISD::ADDCARRY:
754   case ISD::SUBCARRY:
755     if (N->getValueType(0) != MVT::i32)
756       break;
757 
758     SelectAddcSubb(N);
759     return;
760   case ISD::UADDO:
761   case ISD::USUBO: {
762     SelectUADDO_USUBO(N);
763     return;
764   }
765   case AMDGPUISD::FMUL_W_CHAIN: {
766     SelectFMUL_W_CHAIN(N);
767     return;
768   }
769   case AMDGPUISD::FMA_W_CHAIN: {
770     SelectFMA_W_CHAIN(N);
771     return;
772   }
773 
774   case ISD::SCALAR_TO_VECTOR:
775   case ISD::BUILD_VECTOR: {
776     EVT VT = N->getValueType(0);
777     unsigned NumVectorElts = VT.getVectorNumElements();
778     if (VT.getScalarSizeInBits() == 16) {
779       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
780         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
781           ReplaceNode(N, Packed);
782           return;
783         }
784       }
785 
786       break;
787     }
788 
789     assert(VT.getVectorElementType().bitsEq(MVT::i32));
790     unsigned RegClassID =
791         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
792     SelectBuildVector(N, RegClassID);
793     return;
794   }
795   case ISD::BUILD_PAIR: {
796     SDValue RC, SubReg0, SubReg1;
797     SDLoc DL(N);
798     if (N->getValueType(0) == MVT::i128) {
799       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
800       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
801       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
802     } else if (N->getValueType(0) == MVT::i64) {
803       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
804       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
805       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
806     } else {
807       llvm_unreachable("Unhandled value type for BUILD_PAIR");
808     }
809     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
810                             N->getOperand(1), SubReg1 };
811     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
812                                           N->getValueType(0), Ops));
813     return;
814   }
815 
816   case ISD::Constant:
817   case ISD::ConstantFP: {
818     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
819       break;
820 
821     uint64_t Imm;
822     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
823       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
824     else {
825       ConstantSDNode *C = cast<ConstantSDNode>(N);
826       Imm = C->getZExtValue();
827     }
828 
829     SDLoc DL(N);
830     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
831     return;
832   }
833   case AMDGPUISD::BFE_I32:
834   case AMDGPUISD::BFE_U32: {
835     // There is a scalar version available, but unlike the vector version which
836     // has a separate operand for the offset and width, the scalar version packs
837     // the width and offset into a single operand. Try to move to the scalar
838     // version if the offsets are constant, so that we can try to keep extended
839     // loads of kernel arguments in SGPRs.
840 
841     // TODO: Technically we could try to pattern match scalar bitshifts of
842     // dynamic values, but it's probably not useful.
843     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
844     if (!Offset)
845       break;
846 
847     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
848     if (!Width)
849       break;
850 
851     bool Signed = Opc == AMDGPUISD::BFE_I32;
852 
853     uint32_t OffsetVal = Offset->getZExtValue();
854     uint32_t WidthVal = Width->getZExtValue();
855 
856     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
857                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
858     return;
859   }
860   case AMDGPUISD::DIV_SCALE: {
861     SelectDIV_SCALE(N);
862     return;
863   }
864   case AMDGPUISD::MAD_I64_I32:
865   case AMDGPUISD::MAD_U64_U32: {
866     SelectMAD_64_32(N);
867     return;
868   }
869   case ISD::CopyToReg: {
870     const SITargetLowering& Lowering =
871       *static_cast<const SITargetLowering*>(getTargetLowering());
872     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
873     break;
874   }
875   case ISD::AND:
876   case ISD::SRL:
877   case ISD::SRA:
878   case ISD::SIGN_EXTEND_INREG:
879     if (N->getValueType(0) != MVT::i32)
880       break;
881 
882     SelectS_BFE(N);
883     return;
884   case ISD::BRCOND:
885     SelectBRCOND(N);
886     return;
887   case ISD::FMAD:
888   case ISD::FMA:
889     SelectFMAD_FMA(N);
890     return;
891   case AMDGPUISD::ATOMIC_CMP_SWAP:
892     SelectATOMIC_CMP_SWAP(N);
893     return;
894   case AMDGPUISD::CVT_PKRTZ_F16_F32:
895   case AMDGPUISD::CVT_PKNORM_I16_F32:
896   case AMDGPUISD::CVT_PKNORM_U16_F32:
897   case AMDGPUISD::CVT_PK_U16_U32:
898   case AMDGPUISD::CVT_PK_I16_I32: {
899     // Hack around using a legal type if f16 is illegal.
900     if (N->getValueType(0) == MVT::i32) {
901       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
902       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
903                               { N->getOperand(0), N->getOperand(1) });
904       SelectCode(N);
905       return;
906     }
907 
908     break;
909   }
910   case ISD::INTRINSIC_W_CHAIN: {
911     SelectINTRINSIC_W_CHAIN(N);
912     return;
913   }
914   case ISD::INTRINSIC_WO_CHAIN: {
915     SelectINTRINSIC_WO_CHAIN(N);
916     return;
917   }
918   case ISD::INTRINSIC_VOID: {
919     SelectINTRINSIC_VOID(N);
920     return;
921   }
922   }
923 
924   SelectCode(N);
925 }
926 
927 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
928   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
929   const Instruction *Term = BB->getTerminator();
930   return Term->getMetadata("amdgpu.uniform") ||
931          Term->getMetadata("structurizecfg.uniform");
932 }
933 
934 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
935                                           SDValue &N0, SDValue &N1) {
936   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
937       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
938     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
939     // (i64 (bitcast (v2i32 (build_vector
940     //                        (or (extract_vector_elt V, 0), OFFSET),
941     //                        (extract_vector_elt V, 1)))))
942     SDValue Lo = Addr.getOperand(0).getOperand(0);
943     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
944       SDValue BaseLo = Lo.getOperand(0);
945       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
946       // Check that split base (Lo and Hi) are extracted from the same one.
947       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
948           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
949           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
950           // Lo is statically extracted from index 0.
951           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
952           BaseLo.getConstantOperandVal(1) == 0 &&
953           // Hi is statically extracted from index 0.
954           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
955           BaseHi.getConstantOperandVal(1) == 1) {
956         N0 = BaseLo.getOperand(0).getOperand(0);
957         N1 = Lo.getOperand(1);
958         return true;
959       }
960     }
961   }
962   return false;
963 }
964 
965 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
966                                                     SDValue &RHS) const {
967   if (CurDAG->isBaseWithConstantOffset(Addr)) {
968     LHS = Addr.getOperand(0);
969     RHS = Addr.getOperand(1);
970     return true;
971   }
972 
973   if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) {
974     assert(LHS && RHS && isa<ConstantSDNode>(RHS));
975     return true;
976   }
977 
978   return false;
979 }
980 
981 StringRef AMDGPUDAGToDAGISel::getPassName() const {
982   return "AMDGPU DAG->DAG Pattern Instruction Selection";
983 }
984 
985 //===----------------------------------------------------------------------===//
986 // Complex Patterns
987 //===----------------------------------------------------------------------===//
988 
989 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
990                                             SDValue &Offset) {
991   return false;
992 }
993 
994 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
995                                             SDValue &Offset) {
996   ConstantSDNode *C;
997   SDLoc DL(Addr);
998 
999   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
1000     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
1001     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1002   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
1003              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
1004     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
1005     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1006   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
1007             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
1008     Base = Addr.getOperand(0);
1009     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1010   } else {
1011     Base = Addr;
1012     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1013   }
1014 
1015   return true;
1016 }
1017 
1018 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
1019                                                        const SDLoc &DL) const {
1020   SDNode *Mov = CurDAG->getMachineNode(
1021     AMDGPU::S_MOV_B32, DL, MVT::i32,
1022     CurDAG->getTargetConstant(Val, DL, MVT::i32));
1023   return SDValue(Mov, 0);
1024 }
1025 
1026 // FIXME: Should only handle addcarry/subcarry
1027 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
1028   SDLoc DL(N);
1029   SDValue LHS = N->getOperand(0);
1030   SDValue RHS = N->getOperand(1);
1031 
1032   unsigned Opcode = N->getOpcode();
1033   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
1034   bool ProduceCarry =
1035       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
1036   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
1037 
1038   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1039   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1040 
1041   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1042                                        DL, MVT::i32, LHS, Sub0);
1043   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1044                                        DL, MVT::i32, LHS, Sub1);
1045 
1046   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1047                                        DL, MVT::i32, RHS, Sub0);
1048   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1049                                        DL, MVT::i32, RHS, Sub1);
1050 
1051   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
1052 
1053   static const unsigned OpcMap[2][2][2] = {
1054       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
1055        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
1056       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
1057        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
1058 
1059   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
1060   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
1061 
1062   SDNode *AddLo;
1063   if (!ConsumeCarry) {
1064     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1065     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
1066   } else {
1067     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1068     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1069   }
1070   SDValue AddHiArgs[] = {
1071     SDValue(Hi0, 0),
1072     SDValue(Hi1, 0),
1073     SDValue(AddLo, 1)
1074   };
1075   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
1076 
1077   SDValue RegSequenceArgs[] = {
1078     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1079     SDValue(AddLo,0),
1080     Sub0,
1081     SDValue(AddHi,0),
1082     Sub1,
1083   };
1084   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1085                                                MVT::i64, RegSequenceArgs);
1086 
1087   if (ProduceCarry) {
1088     // Replace the carry-use
1089     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1090   }
1091 
1092   // Replace the remaining uses.
1093   ReplaceNode(N, RegSequence);
1094 }
1095 
1096 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1097   SDLoc DL(N);
1098   SDValue LHS = N->getOperand(0);
1099   SDValue RHS = N->getOperand(1);
1100   SDValue CI = N->getOperand(2);
1101 
1102   if (N->isDivergent()) {
1103     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
1104                                                    : AMDGPU::V_SUBB_U32_e64;
1105     CurDAG->SelectNodeTo(
1106         N, Opc, N->getVTList(),
1107         {LHS, RHS, CI,
1108          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1109   } else {
1110     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
1111                                                    : AMDGPU::S_SUB_CO_PSEUDO;
1112     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
1113   }
1114 }
1115 
1116 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1117   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1118   // carry out despite the _i32 name. These were renamed in VI to _U32.
1119   // FIXME: We should probably rename the opcodes here.
1120   bool IsAdd = N->getOpcode() == ISD::UADDO;
1121   bool IsVALU = N->isDivergent();
1122 
1123   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
1124        ++UI)
1125     if (UI.getUse().getResNo() == 1) {
1126       if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
1127           (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
1128         IsVALU = true;
1129         break;
1130       }
1131     }
1132 
1133   if (IsVALU) {
1134     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1135 
1136     CurDAG->SelectNodeTo(
1137         N, Opc, N->getVTList(),
1138         {N->getOperand(0), N->getOperand(1),
1139          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1140   } else {
1141     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
1142                                                 : AMDGPU::S_USUBO_PSEUDO;
1143 
1144     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
1145                          {N->getOperand(0), N->getOperand(1)});
1146   }
1147 }
1148 
1149 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1150   SDLoc SL(N);
1151   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1152   SDValue Ops[10];
1153 
1154   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1155   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1156   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1157   Ops[8] = N->getOperand(0);
1158   Ops[9] = N->getOperand(4);
1159 
1160   CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops);
1161 }
1162 
1163 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1164   SDLoc SL(N);
1165   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
1166   SDValue Ops[8];
1167 
1168   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1169   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1170   Ops[6] = N->getOperand(0);
1171   Ops[7] = N->getOperand(3);
1172 
1173   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1174 }
1175 
1176 // We need to handle this here because tablegen doesn't support matching
1177 // instructions with multiple outputs.
1178 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1179   SDLoc SL(N);
1180   EVT VT = N->getValueType(0);
1181 
1182   assert(VT == MVT::f32 || VT == MVT::f64);
1183 
1184   unsigned Opc
1185     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
1186 
1187   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
1188   // omod
1189   SDValue Ops[8];
1190   SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
1191   SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
1192   SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
1193   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1194 }
1195 
1196 // We need to handle this here because tablegen doesn't support matching
1197 // instructions with multiple outputs.
1198 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1199   SDLoc SL(N);
1200   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1201   unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32;
1202 
1203   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1204   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1205                     Clamp };
1206   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1207 }
1208 
1209 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1210   if (!isUInt<16>(Offset))
1211     return false;
1212 
1213   if (!Base || Subtarget->hasUsableDSOffset() ||
1214       Subtarget->unsafeDSOffsetFoldingEnabled())
1215     return true;
1216 
1217   // On Southern Islands instruction with a negative base value and an offset
1218   // don't seem to work.
1219   return CurDAG->SignBitIsZero(Base);
1220 }
1221 
1222 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1223                                               SDValue &Offset) const {
1224   SDLoc DL(Addr);
1225   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1226     SDValue N0 = Addr.getOperand(0);
1227     SDValue N1 = Addr.getOperand(1);
1228     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1229     if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1230       // (add n0, c0)
1231       Base = N0;
1232       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1233       return true;
1234     }
1235   } else if (Addr.getOpcode() == ISD::SUB) {
1236     // sub C, x -> add (sub 0, x), C
1237     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1238       int64_t ByteOffset = C->getSExtValue();
1239       if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1240         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1241 
1242         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1243         // the known bits in isDSOffsetLegal. We need to emit the selected node
1244         // here, so this is thrown away.
1245         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1246                                       Zero, Addr.getOperand(1));
1247 
1248         if (isDSOffsetLegal(Sub, ByteOffset)) {
1249           SmallVector<SDValue, 3> Opnds;
1250           Opnds.push_back(Zero);
1251           Opnds.push_back(Addr.getOperand(1));
1252 
1253           // FIXME: Select to VOP3 version for with-carry.
1254           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1255           if (Subtarget->hasAddNoCarry()) {
1256             SubOp = AMDGPU::V_SUB_U32_e64;
1257             Opnds.push_back(
1258                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1259           }
1260 
1261           MachineSDNode *MachineSub =
1262               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1263 
1264           Base = SDValue(MachineSub, 0);
1265           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1266           return true;
1267         }
1268       }
1269     }
1270   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1271     // If we have a constant address, prefer to put the constant into the
1272     // offset. This can save moves to load the constant address since multiple
1273     // operations can share the zero base address register, and enables merging
1274     // into read2 / write2 instructions.
1275 
1276     SDLoc DL(Addr);
1277 
1278     if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1279       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1280       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1281                                  DL, MVT::i32, Zero);
1282       Base = SDValue(MovZero, 0);
1283       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1284       return true;
1285     }
1286   }
1287 
1288   // default case
1289   Base = Addr;
1290   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1291   return true;
1292 }
1293 
1294 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1295                                           unsigned Offset1,
1296                                           unsigned Size) const {
1297   if (Offset0 % Size != 0 || Offset1 % Size != 0)
1298     return false;
1299   if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1300     return false;
1301 
1302   if (!Base || Subtarget->hasUsableDSOffset() ||
1303       Subtarget->unsafeDSOffsetFoldingEnabled())
1304     return true;
1305 
1306   // On Southern Islands instruction with a negative base value and an offset
1307   // don't seem to work.
1308   return CurDAG->SignBitIsZero(Base);
1309 }
1310 
1311 // TODO: If offset is too big, put low 16-bit into offset.
1312 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1313                                                    SDValue &Offset0,
1314                                                    SDValue &Offset1) const {
1315   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1316 }
1317 
1318 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1319                                                     SDValue &Offset0,
1320                                                     SDValue &Offset1) const {
1321   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1322 }
1323 
1324 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1325                                             SDValue &Offset0, SDValue &Offset1,
1326                                             unsigned Size) const {
1327   SDLoc DL(Addr);
1328 
1329   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1330     SDValue N0 = Addr.getOperand(0);
1331     SDValue N1 = Addr.getOperand(1);
1332     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1333     unsigned OffsetValue0 = C1->getZExtValue();
1334     unsigned OffsetValue1 = OffsetValue0 + Size;
1335 
1336     // (add n0, c0)
1337     if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1338       Base = N0;
1339       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1340       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1341       return true;
1342     }
1343   } else if (Addr.getOpcode() == ISD::SUB) {
1344     // sub C, x -> add (sub 0, x), C
1345     if (const ConstantSDNode *C =
1346             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1347       unsigned OffsetValue0 = C->getZExtValue();
1348       unsigned OffsetValue1 = OffsetValue0 + Size;
1349 
1350       if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1351         SDLoc DL(Addr);
1352         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1353 
1354         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1355         // the known bits in isDSOffsetLegal. We need to emit the selected node
1356         // here, so this is thrown away.
1357         SDValue Sub =
1358             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1359 
1360         if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1361           SmallVector<SDValue, 3> Opnds;
1362           Opnds.push_back(Zero);
1363           Opnds.push_back(Addr.getOperand(1));
1364           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1365           if (Subtarget->hasAddNoCarry()) {
1366             SubOp = AMDGPU::V_SUB_U32_e64;
1367             Opnds.push_back(
1368                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1369           }
1370 
1371           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1372               SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1373 
1374           Base = SDValue(MachineSub, 0);
1375           Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1376           Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1377           return true;
1378         }
1379       }
1380     }
1381   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1382     unsigned OffsetValue0 = CAddr->getZExtValue();
1383     unsigned OffsetValue1 = OffsetValue0 + Size;
1384 
1385     if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1386       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1387       MachineSDNode *MovZero =
1388           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1389       Base = SDValue(MovZero, 0);
1390       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1391       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1392       return true;
1393     }
1394   }
1395 
1396   // default case
1397 
1398   Base = Addr;
1399   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1400   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1401   return true;
1402 }
1403 
1404 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1405                                      SDValue &VAddr, SDValue &SOffset,
1406                                      SDValue &Offset, SDValue &Offen,
1407                                      SDValue &Idxen, SDValue &Addr64,
1408                                      SDValue &GLC, SDValue &SLC,
1409                                      SDValue &TFE, SDValue &DLC,
1410                                      SDValue &SWZ) const {
1411   // Subtarget prefers to use flat instruction
1412   // FIXME: This should be a pattern predicate and not reach here
1413   if (Subtarget->useFlatForGlobal())
1414     return false;
1415 
1416   SDLoc DL(Addr);
1417 
1418   if (!GLC.getNode())
1419     GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1420   if (!SLC.getNode())
1421     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1422   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
1423   DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1424   SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
1425 
1426   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1427   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1428   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1429   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1430 
1431   ConstantSDNode *C1 = nullptr;
1432   SDValue N0 = Addr;
1433   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1434     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1435     if (isUInt<32>(C1->getZExtValue()))
1436       N0 = Addr.getOperand(0);
1437     else
1438       C1 = nullptr;
1439   }
1440 
1441   if (N0.getOpcode() == ISD::ADD) {
1442     // (add N2, N3) -> addr64, or
1443     // (add (add N2, N3), C1) -> addr64
1444     SDValue N2 = N0.getOperand(0);
1445     SDValue N3 = N0.getOperand(1);
1446     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1447 
1448     if (N2->isDivergent()) {
1449       if (N3->isDivergent()) {
1450         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1451         // addr64, and construct the resource from a 0 address.
1452         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1453         VAddr = N0;
1454       } else {
1455         // N2 is divergent, N3 is not.
1456         Ptr = N3;
1457         VAddr = N2;
1458       }
1459     } else {
1460       // N2 is not divergent.
1461       Ptr = N2;
1462       VAddr = N3;
1463     }
1464     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1465   } else if (N0->isDivergent()) {
1466     // N0 is divergent. Use it as the addr64, and construct the resource from a
1467     // 0 address.
1468     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1469     VAddr = N0;
1470     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1471   } else {
1472     // N0 -> offset, or
1473     // (N0 + C1) -> offset
1474     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1475     Ptr = N0;
1476   }
1477 
1478   if (!C1) {
1479     // No offset.
1480     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1481     return true;
1482   }
1483 
1484   if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1485     // Legal offset for instruction.
1486     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1487     return true;
1488   }
1489 
1490   // Illegal offset, store it in soffset.
1491   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1492   SOffset =
1493       SDValue(CurDAG->getMachineNode(
1494                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1495                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1496               0);
1497   return true;
1498 }
1499 
1500 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1501                                            SDValue &VAddr, SDValue &SOffset,
1502                                            SDValue &Offset, SDValue &GLC,
1503                                            SDValue &SLC, SDValue &TFE,
1504                                            SDValue &DLC, SDValue &SWZ) const {
1505   SDValue Ptr, Offen, Idxen, Addr64;
1506 
1507   // addr64 bit was removed for volcanic islands.
1508   // FIXME: This should be a pattern predicate and not reach here
1509   if (!Subtarget->hasAddr64())
1510     return false;
1511 
1512   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1513               GLC, SLC, TFE, DLC, SWZ))
1514     return false;
1515 
1516   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1517   if (C->getSExtValue()) {
1518     SDLoc DL(Addr);
1519 
1520     const SITargetLowering& Lowering =
1521       *static_cast<const SITargetLowering*>(getTargetLowering());
1522 
1523     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1524     return true;
1525   }
1526 
1527   return false;
1528 }
1529 
1530 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1531                                            SDValue &VAddr, SDValue &SOffset,
1532                                            SDValue &Offset,
1533                                            SDValue &SLC) const {
1534   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1535   SDValue GLC, TFE, DLC, SWZ;
1536 
1537   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1538 }
1539 
1540 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1541   auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1542   return PSV && PSV->isStack();
1543 }
1544 
1545 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1546   SDLoc DL(N);
1547 
1548   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1549   SDValue TFI =
1550       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1551 
1552   // We rebase the base address into an absolute stack address and hence
1553   // use constant 0 for soffset.
1554   return std::make_pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1555 }
1556 
1557 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1558                                                  SDValue Addr, SDValue &Rsrc,
1559                                                  SDValue &VAddr, SDValue &SOffset,
1560                                                  SDValue &ImmOffset) const {
1561 
1562   SDLoc DL(Addr);
1563   MachineFunction &MF = CurDAG->getMachineFunction();
1564   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1565 
1566   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1567 
1568   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1569     int64_t Imm = CAddr->getSExtValue();
1570     const int64_t NullPtr =
1571         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1572     // Don't fold null pointer.
1573     if (Imm != NullPtr) {
1574       SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1575       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1576         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1577       VAddr = SDValue(MovHighBits, 0);
1578 
1579       // In a call sequence, stores to the argument stack area are relative to the
1580       // stack pointer.
1581       const MachinePointerInfo &PtrInfo
1582         = cast<MemSDNode>(Parent)->getPointerInfo();
1583       SOffset = isStackPtrRelative(PtrInfo)
1584         ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1585         : CurDAG->getTargetConstant(0, DL, MVT::i32);
1586       ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1587       return true;
1588     }
1589   }
1590 
1591   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1592     // (add n0, c1)
1593 
1594     SDValue N0 = Addr.getOperand(0);
1595     SDValue N1 = Addr.getOperand(1);
1596 
1597     // Offsets in vaddr must be positive if range checking is enabled.
1598     //
1599     // The total computation of vaddr + soffset + offset must not overflow.  If
1600     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1601     // overflowing.
1602     //
1603     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1604     // always perform a range check. If a negative vaddr base index was used,
1605     // this would fail the range check. The overall address computation would
1606     // compute a valid address, but this doesn't happen due to the range
1607     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1608     //
1609     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1610     // MUBUF vaddr, but not on older subtargets which can only do this if the
1611     // sign bit is known 0.
1612     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1613     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1614         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1615          CurDAG->SignBitIsZero(N0))) {
1616       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1617       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1618       return true;
1619     }
1620   }
1621 
1622   // (node)
1623   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1624   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1625   return true;
1626 }
1627 
1628 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1629                                                   SDValue Addr,
1630                                                   SDValue &SRsrc,
1631                                                   SDValue &SOffset,
1632                                                   SDValue &Offset) const {
1633   ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
1634   if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1635     return false;
1636 
1637   SDLoc DL(Addr);
1638   MachineFunction &MF = CurDAG->getMachineFunction();
1639   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1640 
1641   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1642 
1643   const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
1644 
1645   // FIXME: Get from MachinePointerInfo? We should only be using the frame
1646   // offset if we know this is in a call sequence.
1647   SOffset = isStackPtrRelative(PtrInfo)
1648                 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1649                 : CurDAG->getTargetConstant(0, DL, MVT::i32);
1650 
1651   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1652   return true;
1653 }
1654 
1655 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1656                                            SDValue &SOffset, SDValue &Offset,
1657                                            SDValue &GLC, SDValue &SLC,
1658                                            SDValue &TFE, SDValue &DLC,
1659                                            SDValue &SWZ) const {
1660   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1661   const SIInstrInfo *TII =
1662     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1663 
1664   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1665               GLC, SLC, TFE, DLC, SWZ))
1666     return false;
1667 
1668   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1669       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1670       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1671     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1672                     APInt::getAllOnesValue(32).getZExtValue(); // Size
1673     SDLoc DL(Addr);
1674 
1675     const SITargetLowering& Lowering =
1676       *static_cast<const SITargetLowering*>(getTargetLowering());
1677 
1678     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1679     return true;
1680   }
1681   return false;
1682 }
1683 
1684 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1685                                            SDValue &Soffset, SDValue &Offset
1686                                            ) const {
1687   SDValue GLC, SLC, TFE, DLC, SWZ;
1688 
1689   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1690 }
1691 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1692                                            SDValue &Soffset, SDValue &Offset,
1693                                            SDValue &SLC) const {
1694   SDValue GLC, TFE, DLC, SWZ;
1695 
1696   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1697 }
1698 
1699 // Find a load or store from corresponding pattern root.
1700 // Roots may be build_vector, bitconvert or their combinations.
1701 static MemSDNode* findMemSDNode(SDNode *N) {
1702   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1703   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1704     return MN;
1705   assert(isa<BuildVectorSDNode>(N));
1706   for (SDValue V : N->op_values())
1707     if (MemSDNode *MN =
1708           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1709       return MN;
1710   llvm_unreachable("cannot find MemSDNode in the pattern!");
1711 }
1712 
1713 template <bool IsSigned>
1714 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
1715                                           SDValue Addr,
1716                                           SDValue &VAddr,
1717                                           SDValue &Offset) const {
1718   int64_t OffsetVal = 0;
1719 
1720   unsigned AS = findMemSDNode(N)->getAddressSpace();
1721 
1722   if (Subtarget->hasFlatInstOffsets() &&
1723       (!Subtarget->hasFlatSegmentOffsetBug() ||
1724        AS != AMDGPUAS::FLAT_ADDRESS)) {
1725     SDValue N0, N1;
1726     if (isBaseWithConstantOffset64(Addr, N0, N1)) {
1727       uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1728 
1729       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1730       if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) {
1731         Addr = N0;
1732         OffsetVal = COffsetVal;
1733       } else {
1734         // If the offset doesn't fit, put the low bits into the offset field and
1735         // add the rest.
1736         //
1737         // For a FLAT instruction the hardware decides whether to access
1738         // global/scratch/shared memory based on the high bits of vaddr,
1739         // ignoring the offset field, so we have to ensure that when we add
1740         // remainder to vaddr it still points into the same underlying object.
1741         // The easiest way to do that is to make sure that we split the offset
1742         // into two pieces that are both >= 0 or both <= 0.
1743 
1744         SDLoc DL(N);
1745         uint64_t RemainderOffset;
1746 
1747         std::tie(OffsetVal, RemainderOffset)
1748           = TII->splitFlatOffset(COffsetVal, AS, IsSigned);
1749 
1750         SDValue AddOffsetLo =
1751             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1752         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1753 
1754         if (Addr.getValueType().getSizeInBits() == 32) {
1755           SmallVector<SDValue, 3> Opnds;
1756           Opnds.push_back(N0);
1757           Opnds.push_back(AddOffsetLo);
1758           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1759           if (Subtarget->hasAddNoCarry()) {
1760             AddOp = AMDGPU::V_ADD_U32_e64;
1761             Opnds.push_back(Clamp);
1762           }
1763           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1764         } else {
1765           // TODO: Should this try to use a scalar add pseudo if the base address
1766           // is uniform and saddr is usable?
1767           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1768           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1769 
1770           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1771                                                 DL, MVT::i32, N0, Sub0);
1772           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1773                                                 DL, MVT::i32, N0, Sub1);
1774 
1775           SDValue AddOffsetHi =
1776               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1777 
1778           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1779 
1780           SDNode *Add =
1781               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1782                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1783 
1784           SDNode *Addc = CurDAG->getMachineNode(
1785               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1786               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1787 
1788           SDValue RegSequenceArgs[] = {
1789               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1790               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1791 
1792           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1793                                                 MVT::i64, RegSequenceArgs),
1794                          0);
1795         }
1796       }
1797     }
1798   }
1799 
1800   VAddr = Addr;
1801   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1802   return true;
1803 }
1804 
1805 // If this matches zero_extend i32:x, return x
1806 static SDValue matchZExtFromI32(SDValue Op) {
1807   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1808     return SDValue();
1809 
1810   SDValue ExtSrc = Op.getOperand(0);
1811   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1812 }
1813 
1814 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1815 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1816                                            SDValue Addr,
1817                                            SDValue &SAddr,
1818                                            SDValue &VOffset,
1819                                            SDValue &Offset) const {
1820   int64_t ImmOffset = 0;
1821 
1822   // Match the immediate offset first, which canonically is moved as low as
1823   // possible.
1824   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1825     SDValue LHS = Addr.getOperand(0);
1826     SDValue RHS = Addr.getOperand(1);
1827 
1828     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1829     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1830 
1831     // TODO: Could split larger constant into VGPR offset.
1832     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, true)) {
1833       Addr = LHS;
1834       ImmOffset = COffsetVal;
1835     }
1836   }
1837 
1838   // Match the variable offset.
1839   if (Addr.getOpcode() != ISD::ADD)
1840     return false;
1841 
1842   SDValue LHS = Addr.getOperand(0);
1843   SDValue RHS = Addr.getOperand(1);
1844 
1845   if (!LHS->isDivergent()) {
1846     // add (i64 sgpr), (zero_extend (i32 vgpr))
1847     if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1848       SAddr = LHS;
1849       VOffset = ZextRHS;
1850     }
1851   }
1852 
1853   if (!SAddr && !RHS->isDivergent()) {
1854     // add (zero_extend (i32 vgpr)), (i64 sgpr)
1855     if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1856       SAddr = RHS;
1857       VOffset = ZextLHS;
1858     }
1859   }
1860 
1861   if (!SAddr)
1862     return false;
1863 
1864   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1865   return true;
1866 }
1867 
1868 // Match (32-bit SGPR base) + sext(imm offset)
1869 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *N,
1870                                             SDValue Addr,
1871                                             SDValue &SAddr,
1872                                             SDValue &Offset) const {
1873   if (Addr->isDivergent())
1874     return false;
1875 
1876   SAddr = Addr;
1877   int64_t COffsetVal = 0;
1878 
1879   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1880     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1881     SAddr = Addr.getOperand(0);
1882   }
1883 
1884   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1885     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1886   } else if (SAddr.getOpcode() == ISD::ADD &&
1887              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1888     // Materialize this into a scalar move for scalar address to avoid
1889     // readfirstlane.
1890     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1891     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1892                                               FI->getValueType(0));
1893     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, SDLoc(SAddr),
1894                                            MVT::i32, TFI, SAddr.getOperand(1)),
1895                     0);
1896   }
1897 
1898   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1899 
1900   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1901     int64_t RemainderOffset = COffsetVal;
1902     int64_t ImmField = 0;
1903     const unsigned NumBits = TII->getNumFlatOffsetBits(true);
1904     // Use signed division by a power of two to truncate towards 0.
1905     int64_t D = 1LL << (NumBits - 1);
1906     RemainderOffset = (COffsetVal / D) * D;
1907     ImmField = COffsetVal - RemainderOffset;
1908 
1909     assert(TII->isLegalFLATOffset(ImmField, AMDGPUAS::PRIVATE_ADDRESS, true));
1910     assert(RemainderOffset + ImmField == COffsetVal);
1911 
1912     COffsetVal = ImmField;
1913 
1914     SDLoc DL(N);
1915     SDValue AddOffset =
1916         getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1917     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, DL, MVT::i32,
1918                                            SAddr, AddOffset), 0);
1919   }
1920 
1921   Offset = CurDAG->getTargetConstant(COffsetVal, SDLoc(), MVT::i16);
1922 
1923   return true;
1924 }
1925 
1926 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1927                                           SDValue &Offset, bool &Imm) const {
1928   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1929   if (!C) {
1930     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1931         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1932       Offset = ByteOffsetNode;
1933       Imm = false;
1934       return true;
1935     }
1936     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1937       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1938         Offset = ByteOffsetNode.getOperand(0);
1939         Imm = false;
1940         return true;
1941       }
1942     }
1943     return false;
1944   }
1945 
1946   SDLoc SL(ByteOffsetNode);
1947   // GFX9 and GFX10 have signed byte immediate offsets.
1948   int64_t ByteOffset = C->getSExtValue();
1949   Optional<int64_t> EncodedOffset =
1950       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false);
1951   if (EncodedOffset) {
1952     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1953     Imm = true;
1954     return true;
1955   }
1956 
1957   // SGPR and literal offsets are unsigned.
1958   if (ByteOffset < 0)
1959     return false;
1960 
1961   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1962   if (EncodedOffset) {
1963     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1964     return true;
1965   }
1966 
1967   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1968     return false;
1969 
1970   SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1971   Offset = SDValue(
1972       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1973 
1974   return true;
1975 }
1976 
1977 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1978   if (Addr.getValueType() != MVT::i32)
1979     return Addr;
1980 
1981   // Zero-extend a 32-bit address.
1982   SDLoc SL(Addr);
1983 
1984   const MachineFunction &MF = CurDAG->getMachineFunction();
1985   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1986   unsigned AddrHiVal = Info->get32BitAddressHighBits();
1987   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1988 
1989   const SDValue Ops[] = {
1990     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1991     Addr,
1992     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1993     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1994             0),
1995     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1996   };
1997 
1998   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1999                                         Ops), 0);
2000 }
2001 
2002 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
2003                                      SDValue &Offset, bool &Imm) const {
2004   SDLoc SL(Addr);
2005 
2006   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
2007   // wraparound, because s_load instructions perform the addition in 64 bits.
2008   if ((Addr.getValueType() != MVT::i32 ||
2009        Addr->getFlags().hasNoUnsignedWrap())) {
2010     SDValue N0, N1;
2011     // Extract the base and offset if possible.
2012     if (CurDAG->isBaseWithConstantOffset(Addr) ||
2013         Addr.getOpcode() == ISD::ADD) {
2014       N0 = Addr.getOperand(0);
2015       N1 = Addr.getOperand(1);
2016     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2017       assert(N0 && N1 && isa<ConstantSDNode>(N1));
2018     }
2019     if (N0 && N1) {
2020       if (SelectSMRDOffset(N1, Offset, Imm)) {
2021         SBase = Expand32BitAddress(N0);
2022         return true;
2023       }
2024     }
2025   }
2026   SBase = Expand32BitAddress(Addr);
2027   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
2028   Imm = true;
2029   return true;
2030 }
2031 
2032 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2033                                        SDValue &Offset) const {
2034   bool Imm = false;
2035   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
2036 }
2037 
2038 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2039                                          SDValue &Offset) const {
2040 
2041   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2042 
2043   bool Imm = false;
2044   if (!SelectSMRD(Addr, SBase, Offset, Imm))
2045     return false;
2046 
2047   return !Imm && isa<ConstantSDNode>(Offset);
2048 }
2049 
2050 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2051                                         SDValue &Offset) const {
2052   bool Imm = false;
2053   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
2054          !isa<ConstantSDNode>(Offset);
2055 }
2056 
2057 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
2058                                              SDValue &Offset) const {
2059   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2060     // The immediate offset for S_BUFFER instructions is unsigned.
2061     if (auto Imm =
2062             AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) {
2063       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2064       return true;
2065     }
2066   }
2067 
2068   return false;
2069 }
2070 
2071 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
2072                                                SDValue &Offset) const {
2073   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2074 
2075   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2076     if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget,
2077                                                          C->getZExtValue())) {
2078       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2079       return true;
2080     }
2081   }
2082 
2083   return false;
2084 }
2085 
2086 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2087                                             SDValue &Base,
2088                                             SDValue &Offset) const {
2089   SDLoc DL(Index);
2090 
2091   if (CurDAG->isBaseWithConstantOffset(Index)) {
2092     SDValue N0 = Index.getOperand(0);
2093     SDValue N1 = Index.getOperand(1);
2094     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2095 
2096     // (add n0, c0)
2097     // Don't peel off the offset (c0) if doing so could possibly lead
2098     // the base (n0) to be negative.
2099     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2100     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2101         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2102       Base = N0;
2103       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2104       return true;
2105     }
2106   }
2107 
2108   if (isa<ConstantSDNode>(Index))
2109     return false;
2110 
2111   Base = Index;
2112   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2113   return true;
2114 }
2115 
2116 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
2117                                      SDValue Val, uint32_t Offset,
2118                                      uint32_t Width) {
2119   // Transformation function, pack the offset and width of a BFE into
2120   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2121   // source, bits [5:0] contain the offset and bits [22:16] the width.
2122   uint32_t PackedVal = Offset | (Width << 16);
2123   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2124 
2125   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2126 }
2127 
2128 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2129   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2130   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2131   // Predicate: 0 < b <= c < 32
2132 
2133   const SDValue &Shl = N->getOperand(0);
2134   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2135   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2136 
2137   if (B && C) {
2138     uint32_t BVal = B->getZExtValue();
2139     uint32_t CVal = C->getZExtValue();
2140 
2141     if (0 < BVal && BVal <= CVal && CVal < 32) {
2142       bool Signed = N->getOpcode() == ISD::SRA;
2143       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2144 
2145       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2146                               32 - CVal));
2147       return;
2148     }
2149   }
2150   SelectCode(N);
2151 }
2152 
2153 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2154   switch (N->getOpcode()) {
2155   case ISD::AND:
2156     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2157       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2158       // Predicate: isMask(mask)
2159       const SDValue &Srl = N->getOperand(0);
2160       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2161       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2162 
2163       if (Shift && Mask) {
2164         uint32_t ShiftVal = Shift->getZExtValue();
2165         uint32_t MaskVal = Mask->getZExtValue();
2166 
2167         if (isMask_32(MaskVal)) {
2168           uint32_t WidthVal = countPopulation(MaskVal);
2169 
2170           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2171                                   Srl.getOperand(0), ShiftVal, WidthVal));
2172           return;
2173         }
2174       }
2175     }
2176     break;
2177   case ISD::SRL:
2178     if (N->getOperand(0).getOpcode() == ISD::AND) {
2179       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2180       // Predicate: isMask(mask >> b)
2181       const SDValue &And = N->getOperand(0);
2182       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2183       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2184 
2185       if (Shift && Mask) {
2186         uint32_t ShiftVal = Shift->getZExtValue();
2187         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2188 
2189         if (isMask_32(MaskVal)) {
2190           uint32_t WidthVal = countPopulation(MaskVal);
2191 
2192           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2193                                   And.getOperand(0), ShiftVal, WidthVal));
2194           return;
2195         }
2196       }
2197     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2198       SelectS_BFEFromShifts(N);
2199       return;
2200     }
2201     break;
2202   case ISD::SRA:
2203     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2204       SelectS_BFEFromShifts(N);
2205       return;
2206     }
2207     break;
2208 
2209   case ISD::SIGN_EXTEND_INREG: {
2210     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2211     SDValue Src = N->getOperand(0);
2212     if (Src.getOpcode() != ISD::SRL)
2213       break;
2214 
2215     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2216     if (!Amt)
2217       break;
2218 
2219     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2220     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
2221                             Amt->getZExtValue(), Width));
2222     return;
2223   }
2224   }
2225 
2226   SelectCode(N);
2227 }
2228 
2229 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2230   assert(N->getOpcode() == ISD::BRCOND);
2231   if (!N->hasOneUse())
2232     return false;
2233 
2234   SDValue Cond = N->getOperand(1);
2235   if (Cond.getOpcode() == ISD::CopyToReg)
2236     Cond = Cond.getOperand(2);
2237 
2238   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2239     return false;
2240 
2241   MVT VT = Cond.getOperand(0).getSimpleValueType();
2242   if (VT == MVT::i32)
2243     return true;
2244 
2245   if (VT == MVT::i64) {
2246     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2247 
2248     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2249     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2250   }
2251 
2252   return false;
2253 }
2254 
2255 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2256   SDValue Cond = N->getOperand(1);
2257 
2258   if (Cond.isUndef()) {
2259     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2260                          N->getOperand(2), N->getOperand(0));
2261     return;
2262   }
2263 
2264   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2265   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2266 
2267   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2268   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2269   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2270   SDLoc SL(N);
2271 
2272   if (!UseSCCBr) {
2273     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2274     // analyzed what generates the vcc value, so we do not know whether vcc
2275     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2276     // disabled lanes.
2277     //
2278     // For the case that we select S_CBRANCH_SCC1 and it gets
2279     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2280     // SIInstrInfo::moveToVALU which inserts the S_AND).
2281     //
2282     // We could add an analysis of what generates the vcc value here and omit
2283     // the S_AND when is unnecessary. But it would be better to add a separate
2284     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2285     // catches both cases.
2286     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2287                                                          : AMDGPU::S_AND_B64,
2288                      SL, MVT::i1,
2289                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2290                                                         : AMDGPU::EXEC,
2291                                          MVT::i1),
2292                     Cond),
2293                    0);
2294   }
2295 
2296   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2297   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2298                        N->getOperand(2), // Basic Block
2299                        VCC.getValue(0));
2300 }
2301 
2302 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
2303   MVT VT = N->getSimpleValueType(0);
2304   bool IsFMA = N->getOpcode() == ISD::FMA;
2305   if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
2306                          !Subtarget->hasFmaMixInsts()) ||
2307       ((IsFMA && Subtarget->hasMadMixInsts()) ||
2308        (!IsFMA && Subtarget->hasFmaMixInsts()))) {
2309     SelectCode(N);
2310     return;
2311   }
2312 
2313   SDValue Src0 = N->getOperand(0);
2314   SDValue Src1 = N->getOperand(1);
2315   SDValue Src2 = N->getOperand(2);
2316   unsigned Src0Mods, Src1Mods, Src2Mods;
2317 
2318   // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
2319   // using the conversion from f16.
2320   bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
2321   bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
2322   bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
2323 
2324   assert((IsFMA || !Mode.allFP32Denormals()) &&
2325          "fmad selected with denormals enabled");
2326   // TODO: We can select this with f32 denormals enabled if all the sources are
2327   // converted from f16 (in which case fmad isn't legal).
2328 
2329   if (Sel0 || Sel1 || Sel2) {
2330     // For dummy operands.
2331     SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2332     SDValue Ops[] = {
2333       CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2334       CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2335       CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2336       CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2337       Zero, Zero
2338     };
2339 
2340     CurDAG->SelectNodeTo(N,
2341                          IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2342                          MVT::f32, Ops);
2343   } else {
2344     SelectCode(N);
2345   }
2346 }
2347 
2348 // This is here because there isn't a way to use the generated sub0_sub1 as the
2349 // subreg index to EXTRACT_SUBREG in tablegen.
2350 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2351   MemSDNode *Mem = cast<MemSDNode>(N);
2352   unsigned AS = Mem->getAddressSpace();
2353   if (AS == AMDGPUAS::FLAT_ADDRESS) {
2354     SelectCode(N);
2355     return;
2356   }
2357 
2358   MVT VT = N->getSimpleValueType(0);
2359   bool Is32 = (VT == MVT::i32);
2360   SDLoc SL(N);
2361 
2362   MachineSDNode *CmpSwap = nullptr;
2363   if (Subtarget->hasAddr64()) {
2364     SDValue SRsrc, VAddr, SOffset, Offset, SLC;
2365 
2366     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
2367       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2368         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2369       SDValue CmpVal = Mem->getOperand(2);
2370       SDValue GLC = CurDAG->getTargetConstant(1, SL, MVT::i1);
2371 
2372       // XXX - Do we care about glue operands?
2373 
2374       SDValue Ops[] = {
2375         CmpVal, VAddr, SRsrc, SOffset, Offset, GLC, SLC, Mem->getChain()
2376       };
2377 
2378       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2379     }
2380   }
2381 
2382   if (!CmpSwap) {
2383     SDValue SRsrc, SOffset, Offset, SLC;
2384     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
2385       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2386         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2387 
2388       SDValue CmpVal = Mem->getOperand(2);
2389       SDValue GLC = CurDAG->getTargetConstant(1, SL, MVT::i1);
2390       SDValue Ops[] = {
2391         CmpVal, SRsrc, SOffset, Offset, GLC, SLC, Mem->getChain()
2392       };
2393 
2394       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2395     }
2396   }
2397 
2398   if (!CmpSwap) {
2399     SelectCode(N);
2400     return;
2401   }
2402 
2403   MachineMemOperand *MMO = Mem->getMemOperand();
2404   CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2405 
2406   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2407   SDValue Extract
2408     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2409 
2410   ReplaceUses(SDValue(N, 0), Extract);
2411   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2412   CurDAG->RemoveDeadNode(N);
2413 }
2414 
2415 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2416   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2417   // be copied to an SGPR with readfirstlane.
2418   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2419     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2420 
2421   SDValue Chain = N->getOperand(0);
2422   SDValue Ptr = N->getOperand(2);
2423   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2424   MachineMemOperand *MMO = M->getMemOperand();
2425   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2426 
2427   SDValue Offset;
2428   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2429     SDValue PtrBase = Ptr.getOperand(0);
2430     SDValue PtrOffset = Ptr.getOperand(1);
2431 
2432     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2433     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
2434       N = glueCopyToM0(N, PtrBase);
2435       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2436     }
2437   }
2438 
2439   if (!Offset) {
2440     N = glueCopyToM0(N, Ptr);
2441     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2442   }
2443 
2444   SDValue Ops[] = {
2445     Offset,
2446     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2447     Chain,
2448     N->getOperand(N->getNumOperands() - 1) // New glue
2449   };
2450 
2451   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2452   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2453 }
2454 
2455 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2456   switch (IntrID) {
2457   case Intrinsic::amdgcn_ds_gws_init:
2458     return AMDGPU::DS_GWS_INIT;
2459   case Intrinsic::amdgcn_ds_gws_barrier:
2460     return AMDGPU::DS_GWS_BARRIER;
2461   case Intrinsic::amdgcn_ds_gws_sema_v:
2462     return AMDGPU::DS_GWS_SEMA_V;
2463   case Intrinsic::amdgcn_ds_gws_sema_br:
2464     return AMDGPU::DS_GWS_SEMA_BR;
2465   case Intrinsic::amdgcn_ds_gws_sema_p:
2466     return AMDGPU::DS_GWS_SEMA_P;
2467   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2468     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2469   default:
2470     llvm_unreachable("not a gws intrinsic");
2471   }
2472 }
2473 
2474 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2475   if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2476       !Subtarget->hasGWSSemaReleaseAll()) {
2477     // Let this error.
2478     SelectCode(N);
2479     return;
2480   }
2481 
2482   // Chain, intrinsic ID, vsrc, offset
2483   const bool HasVSrc = N->getNumOperands() == 4;
2484   assert(HasVSrc || N->getNumOperands() == 3);
2485 
2486   SDLoc SL(N);
2487   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2488   int ImmOffset = 0;
2489   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2490   MachineMemOperand *MMO = M->getMemOperand();
2491 
2492   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2493   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2494 
2495   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2496   // offset field) % 64. Some versions of the programming guide omit the m0
2497   // part, or claim it's from offset 0.
2498   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2499     // If we have a constant offset, try to use the 0 in m0 as the base.
2500     // TODO: Look into changing the default m0 initialization value. If the
2501     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2502     // the immediate offset.
2503     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2504     ImmOffset = ConstOffset->getZExtValue();
2505   } else {
2506     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2507       ImmOffset = BaseOffset.getConstantOperandVal(1);
2508       BaseOffset = BaseOffset.getOperand(0);
2509     }
2510 
2511     // Prefer to do the shift in an SGPR since it should be possible to use m0
2512     // as the result directly. If it's already an SGPR, it will be eliminated
2513     // later.
2514     SDNode *SGPROffset
2515       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2516                                BaseOffset);
2517     // Shift to offset in m0
2518     SDNode *M0Base
2519       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2520                                SDValue(SGPROffset, 0),
2521                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2522     glueCopyToM0(N, SDValue(M0Base, 0));
2523   }
2524 
2525   SDValue Chain = N->getOperand(0);
2526   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2527 
2528   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2529   SmallVector<SDValue, 5> Ops;
2530   if (HasVSrc)
2531     Ops.push_back(N->getOperand(2));
2532   Ops.push_back(OffsetField);
2533   Ops.push_back(Chain);
2534 
2535   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2536   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2537 }
2538 
2539 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2540   if (Subtarget->getLDSBankCount() != 16) {
2541     // This is a single instruction with a pattern.
2542     SelectCode(N);
2543     return;
2544   }
2545 
2546   SDLoc DL(N);
2547 
2548   // This requires 2 instructions. It is possible to write a pattern to support
2549   // this, but the generated isel emitter doesn't correctly deal with multiple
2550   // output instructions using the same physical register input. The copy to m0
2551   // is incorrectly placed before the second instruction.
2552   //
2553   // TODO: Match source modifiers.
2554   //
2555   // def : Pat <
2556   //   (int_amdgcn_interp_p1_f16
2557   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2558   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2559   //                             (i1 timm:$high), M0),
2560   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2561   //       timm:$attrchan, 0,
2562   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2563   //   let Predicates = [has16BankLDS];
2564   // }
2565 
2566   // 16 bank LDS
2567   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2568                                       N->getOperand(5), SDValue());
2569 
2570   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2571 
2572   SDNode *InterpMov =
2573     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2574         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2575         N->getOperand(3),  // Attr
2576         N->getOperand(2),  // Attrchan
2577         ToM0.getValue(1) // In glue
2578   });
2579 
2580   SDNode *InterpP1LV =
2581     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2582         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2583         N->getOperand(1), // Src0
2584         N->getOperand(3), // Attr
2585         N->getOperand(2), // Attrchan
2586         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2587         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2588         N->getOperand(4), // high
2589         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2590         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2591         SDValue(InterpMov, 1)
2592   });
2593 
2594   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2595 }
2596 
2597 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2598   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2599   switch (IntrID) {
2600   case Intrinsic::amdgcn_ds_append:
2601   case Intrinsic::amdgcn_ds_consume: {
2602     if (N->getValueType(0) != MVT::i32)
2603       break;
2604     SelectDSAppendConsume(N, IntrID);
2605     return;
2606   }
2607   }
2608 
2609   SelectCode(N);
2610 }
2611 
2612 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2613   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2614   unsigned Opcode;
2615   switch (IntrID) {
2616   case Intrinsic::amdgcn_wqm:
2617     Opcode = AMDGPU::WQM;
2618     break;
2619   case Intrinsic::amdgcn_softwqm:
2620     Opcode = AMDGPU::SOFT_WQM;
2621     break;
2622   case Intrinsic::amdgcn_wwm:
2623     Opcode = AMDGPU::WWM;
2624     break;
2625   case Intrinsic::amdgcn_interp_p1_f16:
2626     SelectInterpP1F16(N);
2627     return;
2628   default:
2629     SelectCode(N);
2630     return;
2631   }
2632 
2633   SDValue Src = N->getOperand(1);
2634   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2635 }
2636 
2637 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2638   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2639   switch (IntrID) {
2640   case Intrinsic::amdgcn_ds_gws_init:
2641   case Intrinsic::amdgcn_ds_gws_barrier:
2642   case Intrinsic::amdgcn_ds_gws_sema_v:
2643   case Intrinsic::amdgcn_ds_gws_sema_br:
2644   case Intrinsic::amdgcn_ds_gws_sema_p:
2645   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2646     SelectDS_GWS(N, IntrID);
2647     return;
2648   default:
2649     break;
2650   }
2651 
2652   SelectCode(N);
2653 }
2654 
2655 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2656                                             unsigned &Mods,
2657                                             bool AllowAbs) const {
2658   Mods = 0;
2659   Src = In;
2660 
2661   if (Src.getOpcode() == ISD::FNEG) {
2662     Mods |= SISrcMods::NEG;
2663     Src = Src.getOperand(0);
2664   }
2665 
2666   if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2667     Mods |= SISrcMods::ABS;
2668     Src = Src.getOperand(0);
2669   }
2670 
2671   return true;
2672 }
2673 
2674 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2675                                         SDValue &SrcMods) const {
2676   unsigned Mods;
2677   if (SelectVOP3ModsImpl(In, Src, Mods)) {
2678     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2679     return true;
2680   }
2681 
2682   return false;
2683 }
2684 
2685 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2686                                          SDValue &SrcMods) const {
2687   unsigned Mods;
2688   if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) {
2689     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2690     return true;
2691   }
2692 
2693   return false;
2694 }
2695 
2696 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2697                                              SDValue &SrcMods) const {
2698   SelectVOP3Mods(In, Src, SrcMods);
2699   return isNoNanSrc(Src);
2700 }
2701 
2702 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2703   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2704     return false;
2705 
2706   Src = In;
2707   return true;
2708 }
2709 
2710 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2711                                          SDValue &SrcMods, SDValue &Clamp,
2712                                          SDValue &Omod) const {
2713   SDLoc DL(In);
2714   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2715   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2716 
2717   return SelectVOP3Mods(In, Src, SrcMods);
2718 }
2719 
2720 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2721                                           SDValue &SrcMods, SDValue &Clamp,
2722                                           SDValue &Omod) const {
2723   SDLoc DL(In);
2724   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2725   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2726 
2727   return SelectVOP3BMods(In, Src, SrcMods);
2728 }
2729 
2730 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2731                                          SDValue &Clamp, SDValue &Omod) const {
2732   Src = In;
2733 
2734   SDLoc DL(In);
2735   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2736   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2737 
2738   return true;
2739 }
2740 
2741 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2742                                          SDValue &SrcMods) const {
2743   unsigned Mods = 0;
2744   Src = In;
2745 
2746   if (Src.getOpcode() == ISD::FNEG) {
2747     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2748     Src = Src.getOperand(0);
2749   }
2750 
2751   if (Src.getOpcode() == ISD::BUILD_VECTOR) {
2752     unsigned VecMods = Mods;
2753 
2754     SDValue Lo = stripBitcast(Src.getOperand(0));
2755     SDValue Hi = stripBitcast(Src.getOperand(1));
2756 
2757     if (Lo.getOpcode() == ISD::FNEG) {
2758       Lo = stripBitcast(Lo.getOperand(0));
2759       Mods ^= SISrcMods::NEG;
2760     }
2761 
2762     if (Hi.getOpcode() == ISD::FNEG) {
2763       Hi = stripBitcast(Hi.getOperand(0));
2764       Mods ^= SISrcMods::NEG_HI;
2765     }
2766 
2767     if (isExtractHiElt(Lo, Lo))
2768       Mods |= SISrcMods::OP_SEL_0;
2769 
2770     if (isExtractHiElt(Hi, Hi))
2771       Mods |= SISrcMods::OP_SEL_1;
2772 
2773     Lo = stripExtractLoElt(Lo);
2774     Hi = stripExtractLoElt(Hi);
2775 
2776     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2777       // Really a scalar input. Just select from the low half of the register to
2778       // avoid packing.
2779 
2780       Src = Lo;
2781       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2782       return true;
2783     }
2784 
2785     Mods = VecMods;
2786   }
2787 
2788   // Packed instructions do not have abs modifiers.
2789   Mods |= SISrcMods::OP_SEL_1;
2790 
2791   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2792   return true;
2793 }
2794 
2795 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2796                                          SDValue &SrcMods) const {
2797   Src = In;
2798   // FIXME: Handle op_sel
2799   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2800   return true;
2801 }
2802 
2803 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2804                                              SDValue &SrcMods) const {
2805   // FIXME: Handle op_sel
2806   return SelectVOP3Mods(In, Src, SrcMods);
2807 }
2808 
2809 // The return value is not whether the match is possible (which it always is),
2810 // but whether or not it a conversion is really used.
2811 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2812                                                    unsigned &Mods) const {
2813   Mods = 0;
2814   SelectVOP3ModsImpl(In, Src, Mods);
2815 
2816   if (Src.getOpcode() == ISD::FP_EXTEND) {
2817     Src = Src.getOperand(0);
2818     assert(Src.getValueType() == MVT::f16);
2819     Src = stripBitcast(Src);
2820 
2821     // Be careful about folding modifiers if we already have an abs. fneg is
2822     // applied last, so we don't want to apply an earlier fneg.
2823     if ((Mods & SISrcMods::ABS) == 0) {
2824       unsigned ModsTmp;
2825       SelectVOP3ModsImpl(Src, Src, ModsTmp);
2826 
2827       if ((ModsTmp & SISrcMods::NEG) != 0)
2828         Mods ^= SISrcMods::NEG;
2829 
2830       if ((ModsTmp & SISrcMods::ABS) != 0)
2831         Mods |= SISrcMods::ABS;
2832     }
2833 
2834     // op_sel/op_sel_hi decide the source type and source.
2835     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2836     // If the sources's op_sel is set, it picks the high half of the source
2837     // register.
2838 
2839     Mods |= SISrcMods::OP_SEL_1;
2840     if (isExtractHiElt(Src, Src)) {
2841       Mods |= SISrcMods::OP_SEL_0;
2842 
2843       // TODO: Should we try to look for neg/abs here?
2844     }
2845 
2846     return true;
2847   }
2848 
2849   return false;
2850 }
2851 
2852 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2853                                                SDValue &SrcMods) const {
2854   unsigned Mods = 0;
2855   SelectVOP3PMadMixModsImpl(In, Src, Mods);
2856   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2857   return true;
2858 }
2859 
2860 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2861   if (In.isUndef())
2862     return CurDAG->getUNDEF(MVT::i32);
2863 
2864   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2865     SDLoc SL(In);
2866     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2867   }
2868 
2869   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2870     SDLoc SL(In);
2871     return CurDAG->getConstant(
2872       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2873   }
2874 
2875   SDValue Src;
2876   if (isExtractHiElt(In, Src))
2877     return Src;
2878 
2879   return SDValue();
2880 }
2881 
2882 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2883   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2884 
2885   const SIRegisterInfo *SIRI =
2886     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2887   const SIInstrInfo * SII =
2888     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2889 
2890   unsigned Limit = 0;
2891   bool AllUsesAcceptSReg = true;
2892   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2893     Limit < 10 && U != E; ++U, ++Limit) {
2894     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2895 
2896     // If the register class is unknown, it could be an unknown
2897     // register class that needs to be an SGPR, e.g. an inline asm
2898     // constraint
2899     if (!RC || SIRI->isSGPRClass(RC))
2900       return false;
2901 
2902     if (RC != &AMDGPU::VS_32RegClass) {
2903       AllUsesAcceptSReg = false;
2904       SDNode * User = *U;
2905       if (User->isMachineOpcode()) {
2906         unsigned Opc = User->getMachineOpcode();
2907         MCInstrDesc Desc = SII->get(Opc);
2908         if (Desc.isCommutable()) {
2909           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2910           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2911           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2912             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2913             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2914             if (CommutedRC == &AMDGPU::VS_32RegClass)
2915               AllUsesAcceptSReg = true;
2916           }
2917         }
2918       }
2919       // If "AllUsesAcceptSReg == false" so far we haven't suceeded
2920       // commuting current user. This means have at least one use
2921       // that strictly require VGPR. Thus, we will not attempt to commute
2922       // other user instructions.
2923       if (!AllUsesAcceptSReg)
2924         break;
2925     }
2926   }
2927   return !AllUsesAcceptSReg && (Limit < 10);
2928 }
2929 
2930 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2931   auto Ld = cast<LoadSDNode>(N);
2932 
2933   return Ld->getAlignment() >= 4 &&
2934         (
2935           (
2936             (
2937               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS       ||
2938               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT
2939             )
2940             &&
2941             !N->isDivergent()
2942           )
2943           ||
2944           (
2945             Subtarget->getScalarizeGlobalBehavior() &&
2946             Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2947             Ld->isSimple() &&
2948             !N->isDivergent() &&
2949             static_cast<const SITargetLowering *>(
2950               getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)
2951           )
2952         );
2953 }
2954 
2955 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2956   const AMDGPUTargetLowering& Lowering =
2957     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2958   bool IsModified = false;
2959   do {
2960     IsModified = false;
2961 
2962     // Go over all selected nodes and try to fold them a bit more
2963     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2964     while (Position != CurDAG->allnodes_end()) {
2965       SDNode *Node = &*Position++;
2966       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
2967       if (!MachineNode)
2968         continue;
2969 
2970       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
2971       if (ResNode != Node) {
2972         if (ResNode)
2973           ReplaceUses(Node, ResNode);
2974         IsModified = true;
2975       }
2976     }
2977     CurDAG->RemoveDeadNodes();
2978   } while (IsModified);
2979 }
2980 
2981 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
2982   Subtarget = &MF.getSubtarget<R600Subtarget>();
2983   return SelectionDAGISel::runOnMachineFunction(MF);
2984 }
2985 
2986 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
2987   if (!N->readMem())
2988     return false;
2989   if (CbId == -1)
2990     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2991            N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
2992 
2993   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
2994 }
2995 
2996 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
2997                                                          SDValue& IntPtr) {
2998   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
2999     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
3000                                        true);
3001     return true;
3002   }
3003   return false;
3004 }
3005 
3006 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
3007     SDValue& BaseReg, SDValue &Offset) {
3008   if (!isa<ConstantSDNode>(Addr)) {
3009     BaseReg = Addr;
3010     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
3011     return true;
3012   }
3013   return false;
3014 }
3015 
3016 void R600DAGToDAGISel::Select(SDNode *N) {
3017   unsigned int Opc = N->getOpcode();
3018   if (N->isMachineOpcode()) {
3019     N->setNodeId(-1);
3020     return;   // Already selected.
3021   }
3022 
3023   switch (Opc) {
3024   default: break;
3025   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
3026   case ISD::SCALAR_TO_VECTOR:
3027   case ISD::BUILD_VECTOR: {
3028     EVT VT = N->getValueType(0);
3029     unsigned NumVectorElts = VT.getVectorNumElements();
3030     unsigned RegClassID;
3031     // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
3032     // that adds a 128 bits reg copy when going through TwoAddressInstructions
3033     // pass. We want to avoid 128 bits copies as much as possible because they
3034     // can't be bundled by our scheduler.
3035     switch(NumVectorElts) {
3036     case 2: RegClassID = R600::R600_Reg64RegClassID; break;
3037     case 4:
3038       if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
3039         RegClassID = R600::R600_Reg128VerticalRegClassID;
3040       else
3041         RegClassID = R600::R600_Reg128RegClassID;
3042       break;
3043     default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
3044     }
3045     SelectBuildVector(N, RegClassID);
3046     return;
3047   }
3048   }
3049 
3050   SelectCode(N);
3051 }
3052 
3053 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
3054                                           SDValue &Offset) {
3055   ConstantSDNode *C;
3056   SDLoc DL(Addr);
3057 
3058   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
3059     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3060     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3061   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
3062              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
3063     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3064     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3065   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
3066             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
3067     Base = Addr.getOperand(0);
3068     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3069   } else {
3070     Base = Addr;
3071     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
3072   }
3073 
3074   return true;
3075 }
3076 
3077 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
3078                                           SDValue &Offset) {
3079   ConstantSDNode *IMMOffset;
3080 
3081   if (Addr.getOpcode() == ISD::ADD
3082       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
3083       && isInt<16>(IMMOffset->getZExtValue())) {
3084 
3085       Base = Addr.getOperand(0);
3086       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3087                                          MVT::i32);
3088       return true;
3089   // If the pointer address is constant, we can move it to the offset field.
3090   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
3091              && isInt<16>(IMMOffset->getZExtValue())) {
3092     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
3093                                   SDLoc(CurDAG->getEntryNode()),
3094                                   R600::ZERO, MVT::i32);
3095     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3096                                        MVT::i32);
3097     return true;
3098   }
3099 
3100   // Default case, no offset
3101   Base = Addr;
3102   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
3103   return true;
3104 }
3105