1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPUArgumentUsageInfo.h"
16 #include "AMDGPUISelLowering.h" // For AMDGPUISD
17 #include "AMDGPUInstrInfo.h"
18 #include "AMDGPUPerfHintAnalysis.h"
19 #include "AMDGPUSubtarget.h"
20 #include "AMDGPUTargetMachine.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "SIDefines.h"
23 #include "SIISelLowering.h"
24 #include "SIInstrInfo.h"
25 #include "SIMachineFunctionInfo.h"
26 #include "SIRegisterInfo.h"
27 #include "llvm/ADT/APInt.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
31 #include "llvm/Analysis/LoopInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/FunctionLoweringInfo.h"
34 #include "llvm/CodeGen/ISDOpcodes.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/SelectionDAG.h"
38 #include "llvm/CodeGen/SelectionDAGISel.h"
39 #include "llvm/CodeGen/SelectionDAGNodes.h"
40 #include "llvm/CodeGen/ValueTypes.h"
41 #include "llvm/IR/BasicBlock.h"
42 #include "llvm/InitializePasses.h"
43 #ifdef EXPENSIVE_CHECKS
44 #include "llvm/IR/Dominators.h"
45 #endif
46 #include "llvm/IR/Instruction.h"
47 #include "llvm/MC/MCInstrDesc.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CodeGen.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/MachineValueType.h"
52 #include "llvm/Support/MathExtras.h"
53 #include <cassert>
54 #include <cstdint>
55 #include <new>
56 #include <vector>
57 
58 #define DEBUG_TYPE "isel"
59 
60 using namespace llvm;
61 
62 namespace llvm {
63 
64 class R600InstrInfo;
65 
66 } // end namespace llvm
67 
68 //===----------------------------------------------------------------------===//
69 // Instruction Selector Implementation
70 //===----------------------------------------------------------------------===//
71 
72 namespace {
73 
74 static bool isNullConstantOrUndef(SDValue V) {
75   if (V.isUndef())
76     return true;
77 
78   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
79   return Const != nullptr && Const->isNullValue();
80 }
81 
82 static bool getConstantValue(SDValue N, uint32_t &Out) {
83   // This is only used for packed vectors, where ussing 0 for undef should
84   // always be good.
85   if (N.isUndef()) {
86     Out = 0;
87     return true;
88   }
89 
90   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
91     Out = C->getAPIntValue().getSExtValue();
92     return true;
93   }
94 
95   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
96     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
97     return true;
98   }
99 
100   return false;
101 }
102 
103 // TODO: Handle undef as zero
104 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
105                                  bool Negate = false) {
106   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
107   uint32_t LHSVal, RHSVal;
108   if (getConstantValue(N->getOperand(0), LHSVal) &&
109       getConstantValue(N->getOperand(1), RHSVal)) {
110     SDLoc SL(N);
111     uint32_t K = Negate ?
112       (-LHSVal & 0xffff) | (-RHSVal << 16) :
113       (LHSVal & 0xffff) | (RHSVal << 16);
114     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
115                               DAG.getTargetConstant(K, SL, MVT::i32));
116   }
117 
118   return nullptr;
119 }
120 
121 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
122   return packConstantV2I16(N, DAG, true);
123 }
124 
125 /// AMDGPU specific code to select AMDGPU machine instructions for
126 /// SelectionDAG operations.
127 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
128   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
129   // make the right decision when generating code for different targets.
130   const GCNSubtarget *Subtarget;
131 
132   // Default FP mode for the current function.
133   AMDGPU::SIModeRegisterDefaults Mode;
134 
135   bool EnableLateStructurizeCFG;
136 
137 public:
138   explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
139                               CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
140     : SelectionDAGISel(*TM, OptLevel) {
141     EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
142   }
143   ~AMDGPUDAGToDAGISel() override = default;
144 
145   void getAnalysisUsage(AnalysisUsage &AU) const override {
146     AU.addRequired<AMDGPUArgumentUsageInfo>();
147     AU.addRequired<LegacyDivergenceAnalysis>();
148 #ifdef EXPENSIVE_CHECKS
149     AU.addRequired<DominatorTreeWrapperPass>();
150     AU.addRequired<LoopInfoWrapperPass>();
151 #endif
152     SelectionDAGISel::getAnalysisUsage(AU);
153   }
154 
155   bool matchLoadD16FromBuildVector(SDNode *N) const;
156 
157   bool runOnMachineFunction(MachineFunction &MF) override;
158   void PreprocessISelDAG() override;
159   void Select(SDNode *N) override;
160   StringRef getPassName() const override;
161   void PostprocessISelDAG() override;
162 
163 protected:
164   void SelectBuildVector(SDNode *N, unsigned RegClassID);
165 
166 private:
167   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
168   bool isNoNanSrc(SDValue N) const;
169   bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
170   bool isNegInlineImmediate(const SDNode *N) const {
171     return isInlineImmediate(N, true);
172   }
173 
174   bool isInlineImmediate16(int64_t Imm) const {
175     return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());
176   }
177 
178   bool isInlineImmediate32(int64_t Imm) const {
179     return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());
180   }
181 
182   bool isInlineImmediate64(int64_t Imm) const {
183     return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());
184   }
185 
186   bool isInlineImmediate(const APFloat &Imm) const {
187     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
188   }
189 
190   bool isVGPRImm(const SDNode *N) const;
191   bool isUniformLoad(const SDNode *N) const;
192   bool isUniformBr(const SDNode *N) const;
193 
194   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
195 
196   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
197   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
198   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
199 
200   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
201   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
202   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
203   bool isDSOffsetLegal(SDValue Base, unsigned Offset,
204                        unsigned OffsetBits) const;
205   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
206   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
207                                  SDValue &Offset1) const;
208   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
209                                   SDValue &Offset1) const;
210   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
211                           SDValue &Offset1, bool IsDS128) const;
212   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
213                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
214                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
215                    SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
216   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
217                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
218                          SDValue &SLC, SDValue &TFE, SDValue &DLC,
219                          SDValue &SWZ) const;
220   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
221                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
222                          SDValue &SLC) const;
223   bool SelectMUBUFScratchOffen(SDNode *Parent,
224                                SDValue Addr, SDValue &RSrc, SDValue &VAddr,
225                                SDValue &SOffset, SDValue &ImmOffset) const;
226   bool SelectMUBUFScratchOffset(SDNode *Parent,
227                                 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
228                                 SDValue &Offset) const;
229 
230   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
231                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
232                          SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
233   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
234                          SDValue &Offset, SDValue &SLC) const;
235   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
236                          SDValue &Offset) const;
237 
238   template <bool IsSigned>
239   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
240                         SDValue &Offset) const;
241   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
242                          SDValue &VOffset, SDValue &Offset) const;
243   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
244                           SDValue &Offset) const;
245 
246   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
247                         bool &Imm) const;
248   SDValue Expand32BitAddress(SDValue Addr) const;
249   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
250                   bool &Imm) const;
251   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
252   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
253   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
254   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
255   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
256   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
257 
258   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
259   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
260                           bool AllowAbs = true) const;
261   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
262   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
263   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
264   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
265                        SDValue &Clamp, SDValue &Omod) const;
266   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
267                         SDValue &Clamp, SDValue &Omod) const;
268   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
269                          SDValue &Clamp, SDValue &Omod) const;
270 
271   bool SelectVOP3OMods(SDValue In, SDValue &Src,
272                        SDValue &Clamp, SDValue &Omod) const;
273 
274   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
275 
276   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
277 
278   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
279   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
280   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
281 
282   SDValue getHi16Elt(SDValue In) const;
283 
284   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
285 
286   void SelectADD_SUB_I64(SDNode *N);
287   void SelectAddcSubb(SDNode *N);
288   void SelectUADDO_USUBO(SDNode *N);
289   void SelectDIV_SCALE(SDNode *N);
290   void SelectMAD_64_32(SDNode *N);
291   void SelectFMA_W_CHAIN(SDNode *N);
292   void SelectFMUL_W_CHAIN(SDNode *N);
293 
294   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
295                    uint32_t Offset, uint32_t Width);
296   void SelectS_BFEFromShifts(SDNode *N);
297   void SelectS_BFE(SDNode *N);
298   bool isCBranchSCC(const SDNode *N) const;
299   void SelectBRCOND(SDNode *N);
300   void SelectFMAD_FMA(SDNode *N);
301   void SelectATOMIC_CMP_SWAP(SDNode *N);
302   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
303   void SelectDS_GWS(SDNode *N, unsigned IntrID);
304   void SelectInterpP1F16(SDNode *N);
305   void SelectINTRINSIC_W_CHAIN(SDNode *N);
306   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
307   void SelectINTRINSIC_VOID(SDNode *N);
308 
309 protected:
310   // Include the pieces autogenerated from the target description.
311 #include "AMDGPUGenDAGISel.inc"
312 };
313 
314 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
315   const R600Subtarget *Subtarget;
316 
317   bool isConstantLoad(const MemSDNode *N, int cbID) const;
318   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
319   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
320                                        SDValue& Offset);
321 public:
322   explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
323       AMDGPUDAGToDAGISel(TM, OptLevel) {}
324 
325   void Select(SDNode *N) override;
326 
327   bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
328                           SDValue &Offset) override;
329   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
330                           SDValue &Offset) override;
331 
332   bool runOnMachineFunction(MachineFunction &MF) override;
333 
334   void PreprocessISelDAG() override {}
335 
336 protected:
337   // Include the pieces autogenerated from the target description.
338 #include "R600GenDAGISel.inc"
339 };
340 
341 static SDValue stripBitcast(SDValue Val) {
342   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
343 }
344 
345 // Figure out if this is really an extract of the high 16-bits of a dword.
346 static bool isExtractHiElt(SDValue In, SDValue &Out) {
347   In = stripBitcast(In);
348   if (In.getOpcode() != ISD::TRUNCATE)
349     return false;
350 
351   SDValue Srl = In.getOperand(0);
352   if (Srl.getOpcode() == ISD::SRL) {
353     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
354       if (ShiftAmt->getZExtValue() == 16) {
355         Out = stripBitcast(Srl.getOperand(0));
356         return true;
357       }
358     }
359   }
360 
361   return false;
362 }
363 
364 // Look through operations that obscure just looking at the low 16-bits of the
365 // same register.
366 static SDValue stripExtractLoElt(SDValue In) {
367   if (In.getOpcode() == ISD::TRUNCATE) {
368     SDValue Src = In.getOperand(0);
369     if (Src.getValueType().getSizeInBits() == 32)
370       return stripBitcast(Src);
371   }
372 
373   return In;
374 }
375 
376 }  // end anonymous namespace
377 
378 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
379                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
380 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
381 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
382 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
383 #ifdef EXPENSIVE_CHECKS
384 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
385 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
386 #endif
387 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
388                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
389 
390 /// This pass converts a legalized DAG into a AMDGPU-specific
391 // DAG, ready for instruction scheduling.
392 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
393                                         CodeGenOpt::Level OptLevel) {
394   return new AMDGPUDAGToDAGISel(TM, OptLevel);
395 }
396 
397 /// This pass converts a legalized DAG into a R600-specific
398 // DAG, ready for instruction scheduling.
399 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
400                                       CodeGenOpt::Level OptLevel) {
401   return new R600DAGToDAGISel(TM, OptLevel);
402 }
403 
404 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
405 #ifdef EXPENSIVE_CHECKS
406   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
407   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
408   for (auto &L : LI->getLoopsInPreorder()) {
409     assert(L->isLCSSAForm(DT));
410   }
411 #endif
412   Subtarget = &MF.getSubtarget<GCNSubtarget>();
413   Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
414   return SelectionDAGISel::runOnMachineFunction(MF);
415 }
416 
417 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
418   assert(Subtarget->d16PreservesUnusedBits());
419   MVT VT = N->getValueType(0).getSimpleVT();
420   if (VT != MVT::v2i16 && VT != MVT::v2f16)
421     return false;
422 
423   SDValue Lo = N->getOperand(0);
424   SDValue Hi = N->getOperand(1);
425 
426   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
427 
428   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
429   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
430   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
431 
432   // Need to check for possible indirect dependencies on the other half of the
433   // vector to avoid introducing a cycle.
434   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
435     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
436 
437     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
438     SDValue Ops[] = {
439       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
440     };
441 
442     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
443     if (LdHi->getMemoryVT() == MVT::i8) {
444       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
445         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
446     } else {
447       assert(LdHi->getMemoryVT() == MVT::i16);
448     }
449 
450     SDValue NewLoadHi =
451       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
452                                   Ops, LdHi->getMemoryVT(),
453                                   LdHi->getMemOperand());
454 
455     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
456     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
457     return true;
458   }
459 
460   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
461   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
462   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
463   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
464   if (LdLo && Lo.hasOneUse()) {
465     SDValue TiedIn = getHi16Elt(Hi);
466     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
467       return false;
468 
469     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
470     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
471     if (LdLo->getMemoryVT() == MVT::i8) {
472       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
473         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
474     } else {
475       assert(LdLo->getMemoryVT() == MVT::i16);
476     }
477 
478     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
479 
480     SDValue Ops[] = {
481       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
482     };
483 
484     SDValue NewLoadLo =
485       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
486                                   Ops, LdLo->getMemoryVT(),
487                                   LdLo->getMemOperand());
488 
489     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
490     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
491     return true;
492   }
493 
494   return false;
495 }
496 
497 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
498   if (!Subtarget->d16PreservesUnusedBits())
499     return;
500 
501   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
502 
503   bool MadeChange = false;
504   while (Position != CurDAG->allnodes_begin()) {
505     SDNode *N = &*--Position;
506     if (N->use_empty())
507       continue;
508 
509     switch (N->getOpcode()) {
510     case ISD::BUILD_VECTOR:
511       MadeChange |= matchLoadD16FromBuildVector(N);
512       break;
513     default:
514       break;
515     }
516   }
517 
518   if (MadeChange) {
519     CurDAG->RemoveDeadNodes();
520     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
521                CurDAG->dump(););
522   }
523 }
524 
525 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
526   if (TM.Options.NoNaNsFPMath)
527     return true;
528 
529   // TODO: Move into isKnownNeverNaN
530   if (N->getFlags().hasNoNaNs())
531     return true;
532 
533   return CurDAG->isKnownNeverNaN(N);
534 }
535 
536 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
537                                            bool Negated) const {
538   if (N->isUndef())
539     return true;
540 
541   const SIInstrInfo *TII = Subtarget->getInstrInfo();
542   if (Negated) {
543     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
544       return TII->isInlineConstant(-C->getAPIntValue());
545 
546     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
547       return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
548 
549   } else {
550     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
551       return TII->isInlineConstant(C->getAPIntValue());
552 
553     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
554       return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
555   }
556 
557   return false;
558 }
559 
560 /// Determine the register class for \p OpNo
561 /// \returns The register class of the virtual register that will be used for
562 /// the given operand number \OpNo or NULL if the register class cannot be
563 /// determined.
564 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
565                                                           unsigned OpNo) const {
566   if (!N->isMachineOpcode()) {
567     if (N->getOpcode() == ISD::CopyToReg) {
568       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
569       if (Reg.isVirtual()) {
570         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
571         return MRI.getRegClass(Reg);
572       }
573 
574       const SIRegisterInfo *TRI
575         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
576       return TRI->getPhysRegClass(Reg);
577     }
578 
579     return nullptr;
580   }
581 
582   switch (N->getMachineOpcode()) {
583   default: {
584     const MCInstrDesc &Desc =
585         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
586     unsigned OpIdx = Desc.getNumDefs() + OpNo;
587     if (OpIdx >= Desc.getNumOperands())
588       return nullptr;
589     int RegClass = Desc.OpInfo[OpIdx].RegClass;
590     if (RegClass == -1)
591       return nullptr;
592 
593     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
594   }
595   case AMDGPU::REG_SEQUENCE: {
596     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
597     const TargetRegisterClass *SuperRC =
598         Subtarget->getRegisterInfo()->getRegClass(RCID);
599 
600     SDValue SubRegOp = N->getOperand(OpNo + 1);
601     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
602     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
603                                                               SubRegIdx);
604   }
605   }
606 }
607 
608 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
609                                          SDValue Glue) const {
610   SmallVector <SDValue, 8> Ops;
611   Ops.push_back(NewChain); // Replace the chain.
612   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
613     Ops.push_back(N->getOperand(i));
614 
615   Ops.push_back(Glue);
616   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
617 }
618 
619 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
620   const SITargetLowering& Lowering =
621     *static_cast<const SITargetLowering*>(getTargetLowering());
622 
623   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
624 
625   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
626   return glueCopyToOp(N, M0, M0.getValue(1));
627 }
628 
629 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
630   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
631   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
632     if (Subtarget->ldsRequiresM0Init())
633       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
634   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
635     MachineFunction &MF = CurDAG->getMachineFunction();
636     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
637     return
638         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
639   }
640   return N;
641 }
642 
643 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
644                                                   EVT VT) const {
645   SDNode *Lo = CurDAG->getMachineNode(
646       AMDGPU::S_MOV_B32, DL, MVT::i32,
647       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
648   SDNode *Hi =
649       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
650                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
651   const SDValue Ops[] = {
652       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
653       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
654       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
655 
656   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
657 }
658 
659 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
660   EVT VT = N->getValueType(0);
661   unsigned NumVectorElts = VT.getVectorNumElements();
662   EVT EltVT = VT.getVectorElementType();
663   SDLoc DL(N);
664   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
665 
666   if (NumVectorElts == 1) {
667     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
668                          RegClass);
669     return;
670   }
671 
672   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
673                                   "supported yet");
674   // 32 = Max Num Vector Elements
675   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
676   // 1 = Vector Register Class
677   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
678 
679   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
680                Triple::amdgcn;
681   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
682   bool IsRegSeq = true;
683   unsigned NOps = N->getNumOperands();
684   for (unsigned i = 0; i < NOps; i++) {
685     // XXX: Why is this here?
686     if (isa<RegisterSDNode>(N->getOperand(i))) {
687       IsRegSeq = false;
688       break;
689     }
690     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
691                          : R600RegisterInfo::getSubRegFromChannel(i);
692     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
693     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
694   }
695   if (NOps != NumVectorElts) {
696     // Fill in the missing undef elements if this was a scalar_to_vector.
697     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
698     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
699                                                    DL, EltVT);
700     for (unsigned i = NOps; i < NumVectorElts; ++i) {
701       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
702                            : R600RegisterInfo::getSubRegFromChannel(i);
703       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
704       RegSeqArgs[1 + (2 * i) + 1] =
705           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
706     }
707   }
708 
709   if (!IsRegSeq)
710     SelectCode(N);
711   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
712 }
713 
714 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
715   unsigned int Opc = N->getOpcode();
716   if (N->isMachineOpcode()) {
717     N->setNodeId(-1);
718     return;   // Already selected.
719   }
720 
721   // isa<MemSDNode> almost works but is slightly too permissive for some DS
722   // intrinsics.
723   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
724       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
725        Opc == ISD::ATOMIC_LOAD_FADD ||
726        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
727        Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
728     N = glueCopyToM0LDSInit(N);
729     SelectCode(N);
730     return;
731   }
732 
733   switch (Opc) {
734   default:
735     break;
736   // We are selecting i64 ADD here instead of custom lower it during
737   // DAG legalization, so we can fold some i64 ADDs used for address
738   // calculation into the LOAD and STORE instructions.
739   case ISD::ADDC:
740   case ISD::ADDE:
741   case ISD::SUBC:
742   case ISD::SUBE: {
743     if (N->getValueType(0) != MVT::i64)
744       break;
745 
746     SelectADD_SUB_I64(N);
747     return;
748   }
749   case ISD::ADDCARRY:
750   case ISD::SUBCARRY:
751     if (N->getValueType(0) != MVT::i32)
752       break;
753 
754     SelectAddcSubb(N);
755     return;
756   case ISD::UADDO:
757   case ISD::USUBO: {
758     SelectUADDO_USUBO(N);
759     return;
760   }
761   case AMDGPUISD::FMUL_W_CHAIN: {
762     SelectFMUL_W_CHAIN(N);
763     return;
764   }
765   case AMDGPUISD::FMA_W_CHAIN: {
766     SelectFMA_W_CHAIN(N);
767     return;
768   }
769 
770   case ISD::SCALAR_TO_VECTOR:
771   case ISD::BUILD_VECTOR: {
772     EVT VT = N->getValueType(0);
773     unsigned NumVectorElts = VT.getVectorNumElements();
774     if (VT.getScalarSizeInBits() == 16) {
775       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
776         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
777           ReplaceNode(N, Packed);
778           return;
779         }
780       }
781 
782       break;
783     }
784 
785     assert(VT.getVectorElementType().bitsEq(MVT::i32));
786     unsigned RegClassID =
787         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
788     SelectBuildVector(N, RegClassID);
789     return;
790   }
791   case ISD::BUILD_PAIR: {
792     SDValue RC, SubReg0, SubReg1;
793     SDLoc DL(N);
794     if (N->getValueType(0) == MVT::i128) {
795       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
796       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
797       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
798     } else if (N->getValueType(0) == MVT::i64) {
799       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
800       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
801       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
802     } else {
803       llvm_unreachable("Unhandled value type for BUILD_PAIR");
804     }
805     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
806                             N->getOperand(1), SubReg1 };
807     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
808                                           N->getValueType(0), Ops));
809     return;
810   }
811 
812   case ISD::Constant:
813   case ISD::ConstantFP: {
814     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
815       break;
816 
817     uint64_t Imm;
818     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
819       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
820     else {
821       ConstantSDNode *C = cast<ConstantSDNode>(N);
822       Imm = C->getZExtValue();
823     }
824 
825     SDLoc DL(N);
826     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
827     return;
828   }
829   case AMDGPUISD::BFE_I32:
830   case AMDGPUISD::BFE_U32: {
831     // There is a scalar version available, but unlike the vector version which
832     // has a separate operand for the offset and width, the scalar version packs
833     // the width and offset into a single operand. Try to move to the scalar
834     // version if the offsets are constant, so that we can try to keep extended
835     // loads of kernel arguments in SGPRs.
836 
837     // TODO: Technically we could try to pattern match scalar bitshifts of
838     // dynamic values, but it's probably not useful.
839     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
840     if (!Offset)
841       break;
842 
843     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
844     if (!Width)
845       break;
846 
847     bool Signed = Opc == AMDGPUISD::BFE_I32;
848 
849     uint32_t OffsetVal = Offset->getZExtValue();
850     uint32_t WidthVal = Width->getZExtValue();
851 
852     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
853                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
854     return;
855   }
856   case AMDGPUISD::DIV_SCALE: {
857     SelectDIV_SCALE(N);
858     return;
859   }
860   case AMDGPUISD::MAD_I64_I32:
861   case AMDGPUISD::MAD_U64_U32: {
862     SelectMAD_64_32(N);
863     return;
864   }
865   case ISD::CopyToReg: {
866     const SITargetLowering& Lowering =
867       *static_cast<const SITargetLowering*>(getTargetLowering());
868     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
869     break;
870   }
871   case ISD::AND:
872   case ISD::SRL:
873   case ISD::SRA:
874   case ISD::SIGN_EXTEND_INREG:
875     if (N->getValueType(0) != MVT::i32)
876       break;
877 
878     SelectS_BFE(N);
879     return;
880   case ISD::BRCOND:
881     SelectBRCOND(N);
882     return;
883   case ISD::FMAD:
884   case ISD::FMA:
885     SelectFMAD_FMA(N);
886     return;
887   case AMDGPUISD::ATOMIC_CMP_SWAP:
888     SelectATOMIC_CMP_SWAP(N);
889     return;
890   case AMDGPUISD::CVT_PKRTZ_F16_F32:
891   case AMDGPUISD::CVT_PKNORM_I16_F32:
892   case AMDGPUISD::CVT_PKNORM_U16_F32:
893   case AMDGPUISD::CVT_PK_U16_U32:
894   case AMDGPUISD::CVT_PK_I16_I32: {
895     // Hack around using a legal type if f16 is illegal.
896     if (N->getValueType(0) == MVT::i32) {
897       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
898       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
899                               { N->getOperand(0), N->getOperand(1) });
900       SelectCode(N);
901       return;
902     }
903 
904     break;
905   }
906   case ISD::INTRINSIC_W_CHAIN: {
907     SelectINTRINSIC_W_CHAIN(N);
908     return;
909   }
910   case ISD::INTRINSIC_WO_CHAIN: {
911     SelectINTRINSIC_WO_CHAIN(N);
912     return;
913   }
914   case ISD::INTRINSIC_VOID: {
915     SelectINTRINSIC_VOID(N);
916     return;
917   }
918   }
919 
920   SelectCode(N);
921 }
922 
923 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
924   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
925   const Instruction *Term = BB->getTerminator();
926   return Term->getMetadata("amdgpu.uniform") ||
927          Term->getMetadata("structurizecfg.uniform");
928 }
929 
930 StringRef AMDGPUDAGToDAGISel::getPassName() const {
931   return "AMDGPU DAG->DAG Pattern Instruction Selection";
932 }
933 
934 //===----------------------------------------------------------------------===//
935 // Complex Patterns
936 //===----------------------------------------------------------------------===//
937 
938 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
939                                             SDValue &Offset) {
940   return false;
941 }
942 
943 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
944                                             SDValue &Offset) {
945   ConstantSDNode *C;
946   SDLoc DL(Addr);
947 
948   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
949     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
950     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
951   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
952              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
953     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
954     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
955   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
956             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
957     Base = Addr.getOperand(0);
958     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
959   } else {
960     Base = Addr;
961     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
962   }
963 
964   return true;
965 }
966 
967 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
968                                                        const SDLoc &DL) const {
969   SDNode *Mov = CurDAG->getMachineNode(
970     AMDGPU::S_MOV_B32, DL, MVT::i32,
971     CurDAG->getTargetConstant(Val, DL, MVT::i32));
972   return SDValue(Mov, 0);
973 }
974 
975 // FIXME: Should only handle addcarry/subcarry
976 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
977   SDLoc DL(N);
978   SDValue LHS = N->getOperand(0);
979   SDValue RHS = N->getOperand(1);
980 
981   unsigned Opcode = N->getOpcode();
982   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
983   bool ProduceCarry =
984       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
985   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
986 
987   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
988   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
989 
990   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
991                                        DL, MVT::i32, LHS, Sub0);
992   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
993                                        DL, MVT::i32, LHS, Sub1);
994 
995   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
996                                        DL, MVT::i32, RHS, Sub0);
997   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
998                                        DL, MVT::i32, RHS, Sub1);
999 
1000   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
1001 
1002   static const unsigned OpcMap[2][2][2] = {
1003       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
1004        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
1005       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
1006        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
1007 
1008   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
1009   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
1010 
1011   SDNode *AddLo;
1012   if (!ConsumeCarry) {
1013     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1014     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
1015   } else {
1016     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1017     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1018   }
1019   SDValue AddHiArgs[] = {
1020     SDValue(Hi0, 0),
1021     SDValue(Hi1, 0),
1022     SDValue(AddLo, 1)
1023   };
1024   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
1025 
1026   SDValue RegSequenceArgs[] = {
1027     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1028     SDValue(AddLo,0),
1029     Sub0,
1030     SDValue(AddHi,0),
1031     Sub1,
1032   };
1033   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1034                                                MVT::i64, RegSequenceArgs);
1035 
1036   if (ProduceCarry) {
1037     // Replace the carry-use
1038     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1039   }
1040 
1041   // Replace the remaining uses.
1042   ReplaceNode(N, RegSequence);
1043 }
1044 
1045 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1046   SDLoc DL(N);
1047   SDValue LHS = N->getOperand(0);
1048   SDValue RHS = N->getOperand(1);
1049   SDValue CI = N->getOperand(2);
1050 
1051   if (N->isDivergent()) {
1052     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
1053                                                    : AMDGPU::V_SUBB_U32_e64;
1054     CurDAG->SelectNodeTo(
1055         N, Opc, N->getVTList(),
1056         {LHS, RHS, CI,
1057          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1058   } else {
1059     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
1060                                                    : AMDGPU::S_SUB_CO_PSEUDO;
1061     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
1062   }
1063 }
1064 
1065 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1066   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1067   // carry out despite the _i32 name. These were renamed in VI to _U32.
1068   // FIXME: We should probably rename the opcodes here.
1069   bool IsAdd = N->getOpcode() == ISD::UADDO;
1070   bool IsVALU = N->isDivergent();
1071 
1072   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
1073        ++UI)
1074     if (UI.getUse().getResNo() == 1) {
1075       if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
1076           (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
1077         IsVALU = true;
1078         break;
1079       }
1080     }
1081 
1082   if (IsVALU) {
1083     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1084 
1085     CurDAG->SelectNodeTo(
1086         N, Opc, N->getVTList(),
1087         {N->getOperand(0), N->getOperand(1),
1088          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1089   } else {
1090     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
1091                                                 : AMDGPU::S_USUBO_PSEUDO;
1092 
1093     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
1094                          {N->getOperand(0), N->getOperand(1)});
1095   }
1096 }
1097 
1098 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1099   SDLoc SL(N);
1100   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1101   SDValue Ops[10];
1102 
1103   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1104   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1105   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1106   Ops[8] = N->getOperand(0);
1107   Ops[9] = N->getOperand(4);
1108 
1109   CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops);
1110 }
1111 
1112 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1113   SDLoc SL(N);
1114   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
1115   SDValue Ops[8];
1116 
1117   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1118   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1119   Ops[6] = N->getOperand(0);
1120   Ops[7] = N->getOperand(3);
1121 
1122   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1123 }
1124 
1125 // We need to handle this here because tablegen doesn't support matching
1126 // instructions with multiple outputs.
1127 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1128   SDLoc SL(N);
1129   EVT VT = N->getValueType(0);
1130 
1131   assert(VT == MVT::f32 || VT == MVT::f64);
1132 
1133   unsigned Opc
1134     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
1135 
1136   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
1137   // omod
1138   SDValue Ops[8];
1139   SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
1140   SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
1141   SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
1142   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1143 }
1144 
1145 // We need to handle this here because tablegen doesn't support matching
1146 // instructions with multiple outputs.
1147 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1148   SDLoc SL(N);
1149   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1150   unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32;
1151 
1152   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1153   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1154                     Clamp };
1155   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1156 }
1157 
1158 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset,
1159                                          unsigned OffsetBits) const {
1160   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
1161       (OffsetBits == 8 && !isUInt<8>(Offset)))
1162     return false;
1163 
1164   if (Subtarget->hasUsableDSOffset() ||
1165       Subtarget->unsafeDSOffsetFoldingEnabled())
1166     return true;
1167 
1168   // On Southern Islands instruction with a negative base value and an offset
1169   // don't seem to work.
1170   return CurDAG->SignBitIsZero(Base);
1171 }
1172 
1173 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1174                                               SDValue &Offset) const {
1175   SDLoc DL(Addr);
1176   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1177     SDValue N0 = Addr.getOperand(0);
1178     SDValue N1 = Addr.getOperand(1);
1179     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1180     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
1181       // (add n0, c0)
1182       Base = N0;
1183       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1184       return true;
1185     }
1186   } else if (Addr.getOpcode() == ISD::SUB) {
1187     // sub C, x -> add (sub 0, x), C
1188     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1189       int64_t ByteOffset = C->getSExtValue();
1190       if (isUInt<16>(ByteOffset)) {
1191         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1192 
1193         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1194         // the known bits in isDSOffsetLegal. We need to emit the selected node
1195         // here, so this is thrown away.
1196         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1197                                       Zero, Addr.getOperand(1));
1198 
1199         if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
1200           SmallVector<SDValue, 3> Opnds;
1201           Opnds.push_back(Zero);
1202           Opnds.push_back(Addr.getOperand(1));
1203 
1204           // FIXME: Select to VOP3 version for with-carry.
1205           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1206           if (Subtarget->hasAddNoCarry()) {
1207             SubOp = AMDGPU::V_SUB_U32_e64;
1208             Opnds.push_back(
1209                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1210           }
1211 
1212           MachineSDNode *MachineSub =
1213               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1214 
1215           Base = SDValue(MachineSub, 0);
1216           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1217           return true;
1218         }
1219       }
1220     }
1221   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1222     // If we have a constant address, prefer to put the constant into the
1223     // offset. This can save moves to load the constant address since multiple
1224     // operations can share the zero base address register, and enables merging
1225     // into read2 / write2 instructions.
1226 
1227     SDLoc DL(Addr);
1228 
1229     if (isUInt<16>(CAddr->getZExtValue())) {
1230       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1231       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1232                                  DL, MVT::i32, Zero);
1233       Base = SDValue(MovZero, 0);
1234       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1235       return true;
1236     }
1237   }
1238 
1239   // default case
1240   Base = Addr;
1241   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1242   return true;
1243 }
1244 
1245 // TODO: If offset is too big, put low 16-bit into offset.
1246 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1247                                                    SDValue &Offset0,
1248                                                    SDValue &Offset1) const {
1249   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, false);
1250 }
1251 
1252 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1253                                                     SDValue &Offset0,
1254                                                     SDValue &Offset1) const {
1255   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, true);
1256 }
1257 
1258 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1259                                             SDValue &Offset0, SDValue &Offset1,
1260                                             bool IsDS128) const {
1261   SDLoc DL(Addr);
1262   unsigned Align = IsDS128 ? 8 : 4;
1263 
1264   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1265     SDValue N0 = Addr.getOperand(0);
1266     SDValue N1 = Addr.getOperand(1);
1267     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1268     unsigned OffsetValue0 = C1->getZExtValue() / Align;
1269     unsigned OffsetValue1 = OffsetValue0 + 1;
1270     // (add n0, c0)
1271     if (isDSOffsetLegal(N0, OffsetValue1, 8)) {
1272       Base = N0;
1273       Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1274       Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1275       return true;
1276     }
1277   } else if (Addr.getOpcode() == ISD::SUB) {
1278     // sub C, x -> add (sub 0, x), C
1279     if (const ConstantSDNode *C =
1280             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1281       unsigned OffsetValue0 = C->getZExtValue() / Align;
1282       unsigned OffsetValue1 = OffsetValue0 + 1;
1283 
1284       if (isUInt<8>(OffsetValue0)) {
1285         SDLoc DL(Addr);
1286         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1287 
1288         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1289         // the known bits in isDSOffsetLegal. We need to emit the selected node
1290         // here, so this is thrown away.
1291         SDValue Sub =
1292             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1293 
1294         if (isDSOffsetLegal(Sub, OffsetValue1, 8)) {
1295           SmallVector<SDValue, 3> Opnds;
1296           Opnds.push_back(Zero);
1297           Opnds.push_back(Addr.getOperand(1));
1298           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1299           if (Subtarget->hasAddNoCarry()) {
1300             SubOp = AMDGPU::V_SUB_U32_e64;
1301             Opnds.push_back(
1302                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1303           }
1304 
1305           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1306               SubOp, DL, (IsDS128 ? MVT::i64 : MVT::i32), Opnds);
1307 
1308           Base = SDValue(MachineSub, 0);
1309           Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1310           Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1311           return true;
1312         }
1313       }
1314     }
1315   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1316     unsigned OffsetValue0 = CAddr->getZExtValue() / Align;
1317     unsigned OffsetValue1 = OffsetValue0 + 1;
1318     bool OffsetIsAligned = Align * OffsetValue0 == CAddr->getZExtValue();
1319 
1320     if (isUInt<8>(OffsetValue0) && isUInt<8>(OffsetValue1) && OffsetIsAligned) {
1321       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1322       MachineSDNode *MovZero =
1323           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1324       Base = SDValue(MovZero, 0);
1325       Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1326       Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1327       return true;
1328     }
1329   }
1330 
1331   // default case
1332 
1333   Base = Addr;
1334   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1335   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1336   return true;
1337 }
1338 
1339 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1340                                      SDValue &VAddr, SDValue &SOffset,
1341                                      SDValue &Offset, SDValue &Offen,
1342                                      SDValue &Idxen, SDValue &Addr64,
1343                                      SDValue &GLC, SDValue &SLC,
1344                                      SDValue &TFE, SDValue &DLC,
1345                                      SDValue &SWZ) const {
1346   // Subtarget prefers to use flat instruction
1347   // FIXME: This should be a pattern predicate and not reach here
1348   if (Subtarget->useFlatForGlobal())
1349     return false;
1350 
1351   SDLoc DL(Addr);
1352 
1353   if (!GLC.getNode())
1354     GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1355   if (!SLC.getNode())
1356     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1357   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
1358   DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1359   SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
1360 
1361   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1362   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1363   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1364   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1365 
1366   ConstantSDNode *C1 = nullptr;
1367   SDValue N0 = Addr;
1368   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1369     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1370     if (isUInt<32>(C1->getZExtValue()))
1371       N0 = Addr.getOperand(0);
1372     else
1373       C1 = nullptr;
1374   }
1375 
1376   if (N0.getOpcode() == ISD::ADD) {
1377     // (add N2, N3) -> addr64, or
1378     // (add (add N2, N3), C1) -> addr64
1379     SDValue N2 = N0.getOperand(0);
1380     SDValue N3 = N0.getOperand(1);
1381     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1382 
1383     if (N2->isDivergent()) {
1384       if (N3->isDivergent()) {
1385         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1386         // addr64, and construct the resource from a 0 address.
1387         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1388         VAddr = N0;
1389       } else {
1390         // N2 is divergent, N3 is not.
1391         Ptr = N3;
1392         VAddr = N2;
1393       }
1394     } else {
1395       // N2 is not divergent.
1396       Ptr = N2;
1397       VAddr = N3;
1398     }
1399     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1400   } else if (N0->isDivergent()) {
1401     // N0 is divergent. Use it as the addr64, and construct the resource from a
1402     // 0 address.
1403     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1404     VAddr = N0;
1405     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1406   } else {
1407     // N0 -> offset, or
1408     // (N0 + C1) -> offset
1409     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1410     Ptr = N0;
1411   }
1412 
1413   if (!C1) {
1414     // No offset.
1415     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1416     return true;
1417   }
1418 
1419   if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1420     // Legal offset for instruction.
1421     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1422     return true;
1423   }
1424 
1425   // Illegal offset, store it in soffset.
1426   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1427   SOffset =
1428       SDValue(CurDAG->getMachineNode(
1429                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1430                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1431               0);
1432   return true;
1433 }
1434 
1435 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1436                                            SDValue &VAddr, SDValue &SOffset,
1437                                            SDValue &Offset, SDValue &GLC,
1438                                            SDValue &SLC, SDValue &TFE,
1439                                            SDValue &DLC, SDValue &SWZ) const {
1440   SDValue Ptr, Offen, Idxen, Addr64;
1441 
1442   // addr64 bit was removed for volcanic islands.
1443   // FIXME: This should be a pattern predicate and not reach here
1444   if (!Subtarget->hasAddr64())
1445     return false;
1446 
1447   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1448               GLC, SLC, TFE, DLC, SWZ))
1449     return false;
1450 
1451   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1452   if (C->getSExtValue()) {
1453     SDLoc DL(Addr);
1454 
1455     const SITargetLowering& Lowering =
1456       *static_cast<const SITargetLowering*>(getTargetLowering());
1457 
1458     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1459     return true;
1460   }
1461 
1462   return false;
1463 }
1464 
1465 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1466                                            SDValue &VAddr, SDValue &SOffset,
1467                                            SDValue &Offset,
1468                                            SDValue &SLC) const {
1469   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1470   SDValue GLC, TFE, DLC, SWZ;
1471 
1472   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1473 }
1474 
1475 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1476   auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1477   return PSV && PSV->isStack();
1478 }
1479 
1480 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1481   SDLoc DL(N);
1482 
1483   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1484   SDValue TFI =
1485       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1486 
1487   // We rebase the base address into an absolute stack address and hence
1488   // use constant 0 for soffset.
1489   return std::make_pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1490 }
1491 
1492 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1493                                                  SDValue Addr, SDValue &Rsrc,
1494                                                  SDValue &VAddr, SDValue &SOffset,
1495                                                  SDValue &ImmOffset) const {
1496 
1497   SDLoc DL(Addr);
1498   MachineFunction &MF = CurDAG->getMachineFunction();
1499   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1500 
1501   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1502 
1503   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1504     int64_t Imm = CAddr->getSExtValue();
1505     const int64_t NullPtr =
1506         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1507     // Don't fold null pointer.
1508     if (Imm != NullPtr) {
1509       SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1510       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1511         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1512       VAddr = SDValue(MovHighBits, 0);
1513 
1514       // In a call sequence, stores to the argument stack area are relative to the
1515       // stack pointer.
1516       const MachinePointerInfo &PtrInfo
1517         = cast<MemSDNode>(Parent)->getPointerInfo();
1518       SOffset = isStackPtrRelative(PtrInfo)
1519         ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1520         : CurDAG->getTargetConstant(0, DL, MVT::i32);
1521       ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1522       return true;
1523     }
1524   }
1525 
1526   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1527     // (add n0, c1)
1528 
1529     SDValue N0 = Addr.getOperand(0);
1530     SDValue N1 = Addr.getOperand(1);
1531 
1532     // Offsets in vaddr must be positive if range checking is enabled.
1533     //
1534     // The total computation of vaddr + soffset + offset must not overflow.  If
1535     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1536     // overflowing.
1537     //
1538     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1539     // always perform a range check. If a negative vaddr base index was used,
1540     // this would fail the range check. The overall address computation would
1541     // compute a valid address, but this doesn't happen due to the range
1542     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1543     //
1544     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1545     // MUBUF vaddr, but not on older subtargets which can only do this if the
1546     // sign bit is known 0.
1547     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1548     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1549         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1550          CurDAG->SignBitIsZero(N0))) {
1551       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1552       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1553       return true;
1554     }
1555   }
1556 
1557   // (node)
1558   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1559   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1560   return true;
1561 }
1562 
1563 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1564                                                   SDValue Addr,
1565                                                   SDValue &SRsrc,
1566                                                   SDValue &SOffset,
1567                                                   SDValue &Offset) const {
1568   ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
1569   if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1570     return false;
1571 
1572   SDLoc DL(Addr);
1573   MachineFunction &MF = CurDAG->getMachineFunction();
1574   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1575 
1576   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1577 
1578   const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
1579 
1580   // FIXME: Get from MachinePointerInfo? We should only be using the frame
1581   // offset if we know this is in a call sequence.
1582   SOffset = isStackPtrRelative(PtrInfo)
1583                 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1584                 : CurDAG->getTargetConstant(0, DL, MVT::i32);
1585 
1586   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1587   return true;
1588 }
1589 
1590 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1591                                            SDValue &SOffset, SDValue &Offset,
1592                                            SDValue &GLC, SDValue &SLC,
1593                                            SDValue &TFE, SDValue &DLC,
1594                                            SDValue &SWZ) const {
1595   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1596   const SIInstrInfo *TII =
1597     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1598 
1599   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1600               GLC, SLC, TFE, DLC, SWZ))
1601     return false;
1602 
1603   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1604       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1605       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1606     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1607                     APInt::getAllOnesValue(32).getZExtValue(); // Size
1608     SDLoc DL(Addr);
1609 
1610     const SITargetLowering& Lowering =
1611       *static_cast<const SITargetLowering*>(getTargetLowering());
1612 
1613     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1614     return true;
1615   }
1616   return false;
1617 }
1618 
1619 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1620                                            SDValue &Soffset, SDValue &Offset
1621                                            ) const {
1622   SDValue GLC, SLC, TFE, DLC, SWZ;
1623 
1624   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1625 }
1626 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1627                                            SDValue &Soffset, SDValue &Offset,
1628                                            SDValue &SLC) const {
1629   SDValue GLC, TFE, DLC, SWZ;
1630 
1631   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1632 }
1633 
1634 // Find a load or store from corresponding pattern root.
1635 // Roots may be build_vector, bitconvert or their combinations.
1636 static MemSDNode* findMemSDNode(SDNode *N) {
1637   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1638   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1639     return MN;
1640   assert(isa<BuildVectorSDNode>(N));
1641   for (SDValue V : N->op_values())
1642     if (MemSDNode *MN =
1643           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1644       return MN;
1645   llvm_unreachable("cannot find MemSDNode in the pattern!");
1646 }
1647 
1648 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
1649                                           SDValue &N0, SDValue &N1) {
1650   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
1651       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
1652     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
1653     // (i64 (bitcast (v2i32 (build_vector
1654     //                        (or (extract_vector_elt V, 0), OFFSET),
1655     //                        (extract_vector_elt V, 1)))))
1656     SDValue Lo = Addr.getOperand(0).getOperand(0);
1657     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
1658       SDValue BaseLo = Lo.getOperand(0);
1659       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
1660       // Check that split base (Lo and Hi) are extracted from the same one.
1661       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1662           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1663           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
1664           // Lo is statically extracted from index 0.
1665           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
1666           BaseLo.getConstantOperandVal(1) == 0 &&
1667           // Hi is statically extracted from index 0.
1668           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
1669           BaseHi.getConstantOperandVal(1) == 1) {
1670         N0 = BaseLo.getOperand(0).getOperand(0);
1671         N1 = Lo.getOperand(1);
1672         return true;
1673       }
1674     }
1675   }
1676   return false;
1677 }
1678 
1679 template <bool IsSigned>
1680 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
1681                                           SDValue Addr,
1682                                           SDValue &VAddr,
1683                                           SDValue &Offset) const {
1684   int64_t OffsetVal = 0;
1685 
1686   unsigned AS = findMemSDNode(N)->getAddressSpace();
1687 
1688   if (Subtarget->hasFlatInstOffsets() &&
1689       (!Subtarget->hasFlatSegmentOffsetBug() ||
1690        AS != AMDGPUAS::FLAT_ADDRESS)) {
1691     SDValue N0, N1;
1692     if (CurDAG->isBaseWithConstantOffset(Addr)) {
1693       N0 = Addr.getOperand(0);
1694       N1 = Addr.getOperand(1);
1695     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
1696       assert(N0 && N1 && isa<ConstantSDNode>(N1));
1697     }
1698     if (N0 && N1) {
1699       uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1700 
1701       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1702       if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) {
1703         Addr = N0;
1704         OffsetVal = COffsetVal;
1705       } else {
1706         // If the offset doesn't fit, put the low bits into the offset field and
1707         // add the rest.
1708         //
1709         // For a FLAT instruction the hardware decides whether to access
1710         // global/scratch/shared memory based on the high bits of vaddr,
1711         // ignoring the offset field, so we have to ensure that when we add
1712         // remainder to vaddr it still points into the same underlying object.
1713         // The easiest way to do that is to make sure that we split the offset
1714         // into two pieces that are both >= 0 or both <= 0.
1715 
1716         SDLoc DL(N);
1717         uint64_t RemainderOffset = COffsetVal;
1718         uint64_t ImmField = 0;
1719         const unsigned NumBits = TII->getNumFlatOffsetBits(IsSigned);
1720         if (IsSigned) {
1721           // Use signed division by a power of two to truncate towards 0.
1722           int64_t D = 1LL << (NumBits - 1);
1723           RemainderOffset = (static_cast<int64_t>(COffsetVal) / D) * D;
1724           ImmField = COffsetVal - RemainderOffset;
1725         } else if (static_cast<int64_t>(COffsetVal) >= 0) {
1726           ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits);
1727           RemainderOffset = COffsetVal - ImmField;
1728         }
1729         assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned));
1730         assert(RemainderOffset + ImmField == COffsetVal);
1731 
1732         OffsetVal = ImmField;
1733 
1734         SDValue AddOffsetLo =
1735             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1736         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1737 
1738         if (Addr.getValueType().getSizeInBits() == 32) {
1739           SmallVector<SDValue, 3> Opnds;
1740           Opnds.push_back(N0);
1741           Opnds.push_back(AddOffsetLo);
1742           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1743           if (Subtarget->hasAddNoCarry()) {
1744             AddOp = AMDGPU::V_ADD_U32_e64;
1745             Opnds.push_back(Clamp);
1746           }
1747           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1748         } else {
1749           // TODO: Should this try to use a scalar add pseudo if the base address
1750           // is uniform and saddr is usable?
1751           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1752           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1753 
1754           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1755                                                 DL, MVT::i32, N0, Sub0);
1756           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1757                                                 DL, MVT::i32, N0, Sub1);
1758 
1759           SDValue AddOffsetHi =
1760               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1761 
1762           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1763 
1764           SDNode *Add =
1765               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1766                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1767 
1768           SDNode *Addc = CurDAG->getMachineNode(
1769               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1770               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1771 
1772           SDValue RegSequenceArgs[] = {
1773               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1774               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1775 
1776           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1777                                                 MVT::i64, RegSequenceArgs),
1778                          0);
1779         }
1780       }
1781     }
1782   }
1783 
1784   VAddr = Addr;
1785   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1786   return true;
1787 }
1788 
1789 // If this matches zero_extend i32:x, return x
1790 static SDValue matchZExtFromI32(SDValue Op) {
1791   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1792     return SDValue();
1793 
1794   SDValue ExtSrc = Op.getOperand(0);
1795   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1796 }
1797 
1798 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1799 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1800                                            SDValue Addr,
1801                                            SDValue &SAddr,
1802                                            SDValue &VOffset,
1803                                            SDValue &Offset) const {
1804   int64_t ImmOffset = 0;
1805 
1806   // Match the immediate offset first, which canonically is moved as low as
1807   // possible.
1808   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1809     SDValue LHS = Addr.getOperand(0);
1810     SDValue RHS = Addr.getOperand(1);
1811 
1812     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1813     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1814 
1815     // TODO: Could split larger constant into VGPR offset.
1816     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, true)) {
1817       Addr = LHS;
1818       ImmOffset = COffsetVal;
1819     }
1820   }
1821 
1822   // Match the variable offset.
1823   if (Addr.getOpcode() != ISD::ADD)
1824     return false;
1825 
1826   SDValue LHS = Addr.getOperand(0);
1827   SDValue RHS = Addr.getOperand(1);
1828 
1829   if (!LHS->isDivergent()) {
1830     // add (i64 sgpr), (zero_extend (i32 vgpr))
1831     if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1832       SAddr = LHS;
1833       VOffset = ZextRHS;
1834     }
1835   }
1836 
1837   if (!SAddr && !RHS->isDivergent()) {
1838     // add (zero_extend (i32 vgpr)), (i64 sgpr)
1839     if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1840       SAddr = RHS;
1841       VOffset = ZextLHS;
1842     }
1843   }
1844 
1845   if (!SAddr)
1846     return false;
1847 
1848   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1849   return true;
1850 }
1851 
1852 // Match (32-bit SGPR base) + sext(imm offset)
1853 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *N,
1854                                             SDValue Addr,
1855                                             SDValue &SAddr,
1856                                             SDValue &Offset) const {
1857   if (Addr->isDivergent())
1858     return false;
1859 
1860   SAddr = Addr;
1861   int64_t COffsetVal = 0;
1862 
1863   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1864     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1865     SAddr = Addr.getOperand(0);
1866   }
1867 
1868   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1869     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1870   } else if (SAddr.getOpcode() == ISD::ADD &&
1871              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1872     // Materialize this into a scalar move for scalar address to avoid
1873     // readfirstlane.
1874     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1875     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1876                                               FI->getValueType(0));
1877     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, SDLoc(SAddr),
1878                                            MVT::i32, TFI, SAddr.getOperand(1)),
1879                     0);
1880   }
1881 
1882   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1883 
1884   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1885     int64_t RemainderOffset = COffsetVal;
1886     int64_t ImmField = 0;
1887     const unsigned NumBits = TII->getNumFlatOffsetBits(true);
1888     // Use signed division by a power of two to truncate towards 0.
1889     int64_t D = 1LL << (NumBits - 1);
1890     RemainderOffset = (COffsetVal / D) * D;
1891     ImmField = COffsetVal - RemainderOffset;
1892 
1893     assert(TII->isLegalFLATOffset(ImmField, AMDGPUAS::PRIVATE_ADDRESS, true));
1894     assert(RemainderOffset + ImmField == COffsetVal);
1895 
1896     COffsetVal = ImmField;
1897 
1898     SDLoc DL(N);
1899     SDValue AddOffset =
1900         getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1901     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, DL, MVT::i32,
1902                                            SAddr, AddOffset), 0);
1903   }
1904 
1905   Offset = CurDAG->getTargetConstant(COffsetVal, SDLoc(), MVT::i16);
1906 
1907   return true;
1908 }
1909 
1910 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1911                                           SDValue &Offset, bool &Imm) const {
1912   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1913   if (!C) {
1914     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1915         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1916       Offset = ByteOffsetNode;
1917       Imm = false;
1918       return true;
1919     }
1920     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1921       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1922         Offset = ByteOffsetNode.getOperand(0);
1923         Imm = false;
1924         return true;
1925       }
1926     }
1927     return false;
1928   }
1929 
1930   SDLoc SL(ByteOffsetNode);
1931   // GFX9 and GFX10 have signed byte immediate offsets.
1932   int64_t ByteOffset = C->getSExtValue();
1933   Optional<int64_t> EncodedOffset =
1934       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false);
1935   if (EncodedOffset) {
1936     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1937     Imm = true;
1938     return true;
1939   }
1940 
1941   // SGPR and literal offsets are unsigned.
1942   if (ByteOffset < 0)
1943     return false;
1944 
1945   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1946   if (EncodedOffset) {
1947     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1948     return true;
1949   }
1950 
1951   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1952     return false;
1953 
1954   SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1955   Offset = SDValue(
1956       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1957 
1958   return true;
1959 }
1960 
1961 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1962   if (Addr.getValueType() != MVT::i32)
1963     return Addr;
1964 
1965   // Zero-extend a 32-bit address.
1966   SDLoc SL(Addr);
1967 
1968   const MachineFunction &MF = CurDAG->getMachineFunction();
1969   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1970   unsigned AddrHiVal = Info->get32BitAddressHighBits();
1971   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1972 
1973   const SDValue Ops[] = {
1974     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1975     Addr,
1976     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1977     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1978             0),
1979     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1980   };
1981 
1982   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1983                                         Ops), 0);
1984 }
1985 
1986 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1987                                      SDValue &Offset, bool &Imm) const {
1988   SDLoc SL(Addr);
1989 
1990   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
1991   // wraparound, because s_load instructions perform the addition in 64 bits.
1992   if ((Addr.getValueType() != MVT::i32 ||
1993        Addr->getFlags().hasNoUnsignedWrap())) {
1994     SDValue N0, N1;
1995     // Extract the base and offset if possible.
1996     if (CurDAG->isBaseWithConstantOffset(Addr) ||
1997         Addr.getOpcode() == ISD::ADD) {
1998       N0 = Addr.getOperand(0);
1999       N1 = Addr.getOperand(1);
2000     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2001       assert(N0 && N1 && isa<ConstantSDNode>(N1));
2002     }
2003     if (N0 && N1) {
2004       if (SelectSMRDOffset(N1, Offset, Imm)) {
2005         SBase = Expand32BitAddress(N0);
2006         return true;
2007       }
2008     }
2009   }
2010   SBase = Expand32BitAddress(Addr);
2011   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
2012   Imm = true;
2013   return true;
2014 }
2015 
2016 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2017                                        SDValue &Offset) const {
2018   bool Imm = false;
2019   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
2020 }
2021 
2022 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2023                                          SDValue &Offset) const {
2024 
2025   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2026 
2027   bool Imm = false;
2028   if (!SelectSMRD(Addr, SBase, Offset, Imm))
2029     return false;
2030 
2031   return !Imm && isa<ConstantSDNode>(Offset);
2032 }
2033 
2034 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2035                                         SDValue &Offset) const {
2036   bool Imm = false;
2037   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
2038          !isa<ConstantSDNode>(Offset);
2039 }
2040 
2041 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
2042                                              SDValue &Offset) const {
2043   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2044     // The immediate offset for S_BUFFER instructions is unsigned.
2045     if (auto Imm =
2046             AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) {
2047       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2048       return true;
2049     }
2050   }
2051 
2052   return false;
2053 }
2054 
2055 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
2056                                                SDValue &Offset) const {
2057   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2058 
2059   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2060     if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget,
2061                                                          C->getZExtValue())) {
2062       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2063       return true;
2064     }
2065   }
2066 
2067   return false;
2068 }
2069 
2070 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2071                                             SDValue &Base,
2072                                             SDValue &Offset) const {
2073   SDLoc DL(Index);
2074 
2075   if (CurDAG->isBaseWithConstantOffset(Index)) {
2076     SDValue N0 = Index.getOperand(0);
2077     SDValue N1 = Index.getOperand(1);
2078     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2079 
2080     // (add n0, c0)
2081     // Don't peel off the offset (c0) if doing so could possibly lead
2082     // the base (n0) to be negative.
2083     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2084     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2085         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2086       Base = N0;
2087       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2088       return true;
2089     }
2090   }
2091 
2092   if (isa<ConstantSDNode>(Index))
2093     return false;
2094 
2095   Base = Index;
2096   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2097   return true;
2098 }
2099 
2100 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
2101                                      SDValue Val, uint32_t Offset,
2102                                      uint32_t Width) {
2103   // Transformation function, pack the offset and width of a BFE into
2104   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2105   // source, bits [5:0] contain the offset and bits [22:16] the width.
2106   uint32_t PackedVal = Offset | (Width << 16);
2107   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2108 
2109   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2110 }
2111 
2112 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2113   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2114   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2115   // Predicate: 0 < b <= c < 32
2116 
2117   const SDValue &Shl = N->getOperand(0);
2118   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2119   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2120 
2121   if (B && C) {
2122     uint32_t BVal = B->getZExtValue();
2123     uint32_t CVal = C->getZExtValue();
2124 
2125     if (0 < BVal && BVal <= CVal && CVal < 32) {
2126       bool Signed = N->getOpcode() == ISD::SRA;
2127       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2128 
2129       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2130                               32 - CVal));
2131       return;
2132     }
2133   }
2134   SelectCode(N);
2135 }
2136 
2137 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2138   switch (N->getOpcode()) {
2139   case ISD::AND:
2140     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2141       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2142       // Predicate: isMask(mask)
2143       const SDValue &Srl = N->getOperand(0);
2144       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2145       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2146 
2147       if (Shift && Mask) {
2148         uint32_t ShiftVal = Shift->getZExtValue();
2149         uint32_t MaskVal = Mask->getZExtValue();
2150 
2151         if (isMask_32(MaskVal)) {
2152           uint32_t WidthVal = countPopulation(MaskVal);
2153 
2154           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2155                                   Srl.getOperand(0), ShiftVal, WidthVal));
2156           return;
2157         }
2158       }
2159     }
2160     break;
2161   case ISD::SRL:
2162     if (N->getOperand(0).getOpcode() == ISD::AND) {
2163       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2164       // Predicate: isMask(mask >> b)
2165       const SDValue &And = N->getOperand(0);
2166       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2167       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2168 
2169       if (Shift && Mask) {
2170         uint32_t ShiftVal = Shift->getZExtValue();
2171         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2172 
2173         if (isMask_32(MaskVal)) {
2174           uint32_t WidthVal = countPopulation(MaskVal);
2175 
2176           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2177                                   And.getOperand(0), ShiftVal, WidthVal));
2178           return;
2179         }
2180       }
2181     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2182       SelectS_BFEFromShifts(N);
2183       return;
2184     }
2185     break;
2186   case ISD::SRA:
2187     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2188       SelectS_BFEFromShifts(N);
2189       return;
2190     }
2191     break;
2192 
2193   case ISD::SIGN_EXTEND_INREG: {
2194     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2195     SDValue Src = N->getOperand(0);
2196     if (Src.getOpcode() != ISD::SRL)
2197       break;
2198 
2199     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2200     if (!Amt)
2201       break;
2202 
2203     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2204     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
2205                             Amt->getZExtValue(), Width));
2206     return;
2207   }
2208   }
2209 
2210   SelectCode(N);
2211 }
2212 
2213 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2214   assert(N->getOpcode() == ISD::BRCOND);
2215   if (!N->hasOneUse())
2216     return false;
2217 
2218   SDValue Cond = N->getOperand(1);
2219   if (Cond.getOpcode() == ISD::CopyToReg)
2220     Cond = Cond.getOperand(2);
2221 
2222   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2223     return false;
2224 
2225   MVT VT = Cond.getOperand(0).getSimpleValueType();
2226   if (VT == MVT::i32)
2227     return true;
2228 
2229   if (VT == MVT::i64) {
2230     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2231 
2232     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2233     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2234   }
2235 
2236   return false;
2237 }
2238 
2239 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2240   SDValue Cond = N->getOperand(1);
2241 
2242   if (Cond.isUndef()) {
2243     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2244                          N->getOperand(2), N->getOperand(0));
2245     return;
2246   }
2247 
2248   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2249   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2250 
2251   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2252   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2253   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2254   SDLoc SL(N);
2255 
2256   if (!UseSCCBr) {
2257     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2258     // analyzed what generates the vcc value, so we do not know whether vcc
2259     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2260     // disabled lanes.
2261     //
2262     // For the case that we select S_CBRANCH_SCC1 and it gets
2263     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2264     // SIInstrInfo::moveToVALU which inserts the S_AND).
2265     //
2266     // We could add an analysis of what generates the vcc value here and omit
2267     // the S_AND when is unnecessary. But it would be better to add a separate
2268     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2269     // catches both cases.
2270     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2271                                                          : AMDGPU::S_AND_B64,
2272                      SL, MVT::i1,
2273                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2274                                                         : AMDGPU::EXEC,
2275                                          MVT::i1),
2276                     Cond),
2277                    0);
2278   }
2279 
2280   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2281   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2282                        N->getOperand(2), // Basic Block
2283                        VCC.getValue(0));
2284 }
2285 
2286 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
2287   MVT VT = N->getSimpleValueType(0);
2288   bool IsFMA = N->getOpcode() == ISD::FMA;
2289   if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
2290                          !Subtarget->hasFmaMixInsts()) ||
2291       ((IsFMA && Subtarget->hasMadMixInsts()) ||
2292        (!IsFMA && Subtarget->hasFmaMixInsts()))) {
2293     SelectCode(N);
2294     return;
2295   }
2296 
2297   SDValue Src0 = N->getOperand(0);
2298   SDValue Src1 = N->getOperand(1);
2299   SDValue Src2 = N->getOperand(2);
2300   unsigned Src0Mods, Src1Mods, Src2Mods;
2301 
2302   // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
2303   // using the conversion from f16.
2304   bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
2305   bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
2306   bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
2307 
2308   assert((IsFMA || !Mode.allFP32Denormals()) &&
2309          "fmad selected with denormals enabled");
2310   // TODO: We can select this with f32 denormals enabled if all the sources are
2311   // converted from f16 (in which case fmad isn't legal).
2312 
2313   if (Sel0 || Sel1 || Sel2) {
2314     // For dummy operands.
2315     SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2316     SDValue Ops[] = {
2317       CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2318       CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2319       CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2320       CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2321       Zero, Zero
2322     };
2323 
2324     CurDAG->SelectNodeTo(N,
2325                          IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2326                          MVT::f32, Ops);
2327   } else {
2328     SelectCode(N);
2329   }
2330 }
2331 
2332 // This is here because there isn't a way to use the generated sub0_sub1 as the
2333 // subreg index to EXTRACT_SUBREG in tablegen.
2334 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2335   MemSDNode *Mem = cast<MemSDNode>(N);
2336   unsigned AS = Mem->getAddressSpace();
2337   if (AS == AMDGPUAS::FLAT_ADDRESS) {
2338     SelectCode(N);
2339     return;
2340   }
2341 
2342   MVT VT = N->getSimpleValueType(0);
2343   bool Is32 = (VT == MVT::i32);
2344   SDLoc SL(N);
2345 
2346   MachineSDNode *CmpSwap = nullptr;
2347   if (Subtarget->hasAddr64()) {
2348     SDValue SRsrc, VAddr, SOffset, Offset, SLC;
2349 
2350     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
2351       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2352         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2353       SDValue CmpVal = Mem->getOperand(2);
2354 
2355       // XXX - Do we care about glue operands?
2356 
2357       SDValue Ops[] = {
2358         CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2359       };
2360 
2361       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2362     }
2363   }
2364 
2365   if (!CmpSwap) {
2366     SDValue SRsrc, SOffset, Offset, SLC;
2367     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
2368       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2369         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2370 
2371       SDValue CmpVal = Mem->getOperand(2);
2372       SDValue Ops[] = {
2373         CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2374       };
2375 
2376       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2377     }
2378   }
2379 
2380   if (!CmpSwap) {
2381     SelectCode(N);
2382     return;
2383   }
2384 
2385   MachineMemOperand *MMO = Mem->getMemOperand();
2386   CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2387 
2388   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2389   SDValue Extract
2390     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2391 
2392   ReplaceUses(SDValue(N, 0), Extract);
2393   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2394   CurDAG->RemoveDeadNode(N);
2395 }
2396 
2397 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2398   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2399   // be copied to an SGPR with readfirstlane.
2400   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2401     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2402 
2403   SDValue Chain = N->getOperand(0);
2404   SDValue Ptr = N->getOperand(2);
2405   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2406   MachineMemOperand *MMO = M->getMemOperand();
2407   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2408 
2409   SDValue Offset;
2410   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2411     SDValue PtrBase = Ptr.getOperand(0);
2412     SDValue PtrOffset = Ptr.getOperand(1);
2413 
2414     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2415     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) {
2416       N = glueCopyToM0(N, PtrBase);
2417       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2418     }
2419   }
2420 
2421   if (!Offset) {
2422     N = glueCopyToM0(N, Ptr);
2423     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2424   }
2425 
2426   SDValue Ops[] = {
2427     Offset,
2428     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2429     Chain,
2430     N->getOperand(N->getNumOperands() - 1) // New glue
2431   };
2432 
2433   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2434   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2435 }
2436 
2437 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2438   switch (IntrID) {
2439   case Intrinsic::amdgcn_ds_gws_init:
2440     return AMDGPU::DS_GWS_INIT;
2441   case Intrinsic::amdgcn_ds_gws_barrier:
2442     return AMDGPU::DS_GWS_BARRIER;
2443   case Intrinsic::amdgcn_ds_gws_sema_v:
2444     return AMDGPU::DS_GWS_SEMA_V;
2445   case Intrinsic::amdgcn_ds_gws_sema_br:
2446     return AMDGPU::DS_GWS_SEMA_BR;
2447   case Intrinsic::amdgcn_ds_gws_sema_p:
2448     return AMDGPU::DS_GWS_SEMA_P;
2449   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2450     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2451   default:
2452     llvm_unreachable("not a gws intrinsic");
2453   }
2454 }
2455 
2456 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2457   if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2458       !Subtarget->hasGWSSemaReleaseAll()) {
2459     // Let this error.
2460     SelectCode(N);
2461     return;
2462   }
2463 
2464   // Chain, intrinsic ID, vsrc, offset
2465   const bool HasVSrc = N->getNumOperands() == 4;
2466   assert(HasVSrc || N->getNumOperands() == 3);
2467 
2468   SDLoc SL(N);
2469   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2470   int ImmOffset = 0;
2471   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2472   MachineMemOperand *MMO = M->getMemOperand();
2473 
2474   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2475   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2476 
2477   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2478   // offset field) % 64. Some versions of the programming guide omit the m0
2479   // part, or claim it's from offset 0.
2480   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2481     // If we have a constant offset, try to use the 0 in m0 as the base.
2482     // TODO: Look into changing the default m0 initialization value. If the
2483     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2484     // the immediate offset.
2485     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2486     ImmOffset = ConstOffset->getZExtValue();
2487   } else {
2488     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2489       ImmOffset = BaseOffset.getConstantOperandVal(1);
2490       BaseOffset = BaseOffset.getOperand(0);
2491     }
2492 
2493     // Prefer to do the shift in an SGPR since it should be possible to use m0
2494     // as the result directly. If it's already an SGPR, it will be eliminated
2495     // later.
2496     SDNode *SGPROffset
2497       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2498                                BaseOffset);
2499     // Shift to offset in m0
2500     SDNode *M0Base
2501       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2502                                SDValue(SGPROffset, 0),
2503                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2504     glueCopyToM0(N, SDValue(M0Base, 0));
2505   }
2506 
2507   SDValue Chain = N->getOperand(0);
2508   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2509 
2510   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2511   SmallVector<SDValue, 5> Ops;
2512   if (HasVSrc)
2513     Ops.push_back(N->getOperand(2));
2514   Ops.push_back(OffsetField);
2515   Ops.push_back(Chain);
2516 
2517   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2518   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2519 }
2520 
2521 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2522   if (Subtarget->getLDSBankCount() != 16) {
2523     // This is a single instruction with a pattern.
2524     SelectCode(N);
2525     return;
2526   }
2527 
2528   SDLoc DL(N);
2529 
2530   // This requires 2 instructions. It is possible to write a pattern to support
2531   // this, but the generated isel emitter doesn't correctly deal with multiple
2532   // output instructions using the same physical register input. The copy to m0
2533   // is incorrectly placed before the second instruction.
2534   //
2535   // TODO: Match source modifiers.
2536   //
2537   // def : Pat <
2538   //   (int_amdgcn_interp_p1_f16
2539   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2540   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2541   //                             (i1 timm:$high), M0),
2542   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2543   //       timm:$attrchan, 0,
2544   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2545   //   let Predicates = [has16BankLDS];
2546   // }
2547 
2548   // 16 bank LDS
2549   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2550                                       N->getOperand(5), SDValue());
2551 
2552   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2553 
2554   SDNode *InterpMov =
2555     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2556         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2557         N->getOperand(3),  // Attr
2558         N->getOperand(2),  // Attrchan
2559         ToM0.getValue(1) // In glue
2560   });
2561 
2562   SDNode *InterpP1LV =
2563     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2564         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2565         N->getOperand(1), // Src0
2566         N->getOperand(3), // Attr
2567         N->getOperand(2), // Attrchan
2568         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2569         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2570         N->getOperand(4), // high
2571         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2572         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2573         SDValue(InterpMov, 1)
2574   });
2575 
2576   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2577 }
2578 
2579 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2580   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2581   switch (IntrID) {
2582   case Intrinsic::amdgcn_ds_append:
2583   case Intrinsic::amdgcn_ds_consume: {
2584     if (N->getValueType(0) != MVT::i32)
2585       break;
2586     SelectDSAppendConsume(N, IntrID);
2587     return;
2588   }
2589   }
2590 
2591   SelectCode(N);
2592 }
2593 
2594 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2595   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2596   unsigned Opcode;
2597   switch (IntrID) {
2598   case Intrinsic::amdgcn_wqm:
2599     Opcode = AMDGPU::WQM;
2600     break;
2601   case Intrinsic::amdgcn_softwqm:
2602     Opcode = AMDGPU::SOFT_WQM;
2603     break;
2604   case Intrinsic::amdgcn_wwm:
2605     Opcode = AMDGPU::WWM;
2606     break;
2607   case Intrinsic::amdgcn_interp_p1_f16:
2608     SelectInterpP1F16(N);
2609     return;
2610   default:
2611     SelectCode(N);
2612     return;
2613   }
2614 
2615   SDValue Src = N->getOperand(1);
2616   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2617 }
2618 
2619 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2620   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2621   switch (IntrID) {
2622   case Intrinsic::amdgcn_ds_gws_init:
2623   case Intrinsic::amdgcn_ds_gws_barrier:
2624   case Intrinsic::amdgcn_ds_gws_sema_v:
2625   case Intrinsic::amdgcn_ds_gws_sema_br:
2626   case Intrinsic::amdgcn_ds_gws_sema_p:
2627   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2628     SelectDS_GWS(N, IntrID);
2629     return;
2630   default:
2631     break;
2632   }
2633 
2634   SelectCode(N);
2635 }
2636 
2637 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2638                                             unsigned &Mods,
2639                                             bool AllowAbs) const {
2640   Mods = 0;
2641   Src = In;
2642 
2643   if (Src.getOpcode() == ISD::FNEG) {
2644     Mods |= SISrcMods::NEG;
2645     Src = Src.getOperand(0);
2646   }
2647 
2648   if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2649     Mods |= SISrcMods::ABS;
2650     Src = Src.getOperand(0);
2651   }
2652 
2653   return true;
2654 }
2655 
2656 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2657                                         SDValue &SrcMods) const {
2658   unsigned Mods;
2659   if (SelectVOP3ModsImpl(In, Src, Mods)) {
2660     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2661     return true;
2662   }
2663 
2664   return false;
2665 }
2666 
2667 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2668                                          SDValue &SrcMods) const {
2669   unsigned Mods;
2670   if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) {
2671     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2672     return true;
2673   }
2674 
2675   return false;
2676 }
2677 
2678 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2679                                              SDValue &SrcMods) const {
2680   SelectVOP3Mods(In, Src, SrcMods);
2681   return isNoNanSrc(Src);
2682 }
2683 
2684 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2685   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2686     return false;
2687 
2688   Src = In;
2689   return true;
2690 }
2691 
2692 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2693                                          SDValue &SrcMods, SDValue &Clamp,
2694                                          SDValue &Omod) const {
2695   SDLoc DL(In);
2696   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2697   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2698 
2699   return SelectVOP3Mods(In, Src, SrcMods);
2700 }
2701 
2702 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2703                                           SDValue &SrcMods, SDValue &Clamp,
2704                                           SDValue &Omod) const {
2705   SDLoc DL(In);
2706   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2707   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2708 
2709   return SelectVOP3BMods(In, Src, SrcMods);
2710 }
2711 
2712 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2713                                          SDValue &Clamp, SDValue &Omod) const {
2714   Src = In;
2715 
2716   SDLoc DL(In);
2717   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2718   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2719 
2720   return true;
2721 }
2722 
2723 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2724                                          SDValue &SrcMods) const {
2725   unsigned Mods = 0;
2726   Src = In;
2727 
2728   if (Src.getOpcode() == ISD::FNEG) {
2729     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2730     Src = Src.getOperand(0);
2731   }
2732 
2733   if (Src.getOpcode() == ISD::BUILD_VECTOR) {
2734     unsigned VecMods = Mods;
2735 
2736     SDValue Lo = stripBitcast(Src.getOperand(0));
2737     SDValue Hi = stripBitcast(Src.getOperand(1));
2738 
2739     if (Lo.getOpcode() == ISD::FNEG) {
2740       Lo = stripBitcast(Lo.getOperand(0));
2741       Mods ^= SISrcMods::NEG;
2742     }
2743 
2744     if (Hi.getOpcode() == ISD::FNEG) {
2745       Hi = stripBitcast(Hi.getOperand(0));
2746       Mods ^= SISrcMods::NEG_HI;
2747     }
2748 
2749     if (isExtractHiElt(Lo, Lo))
2750       Mods |= SISrcMods::OP_SEL_0;
2751 
2752     if (isExtractHiElt(Hi, Hi))
2753       Mods |= SISrcMods::OP_SEL_1;
2754 
2755     Lo = stripExtractLoElt(Lo);
2756     Hi = stripExtractLoElt(Hi);
2757 
2758     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2759       // Really a scalar input. Just select from the low half of the register to
2760       // avoid packing.
2761 
2762       Src = Lo;
2763       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2764       return true;
2765     }
2766 
2767     Mods = VecMods;
2768   }
2769 
2770   // Packed instructions do not have abs modifiers.
2771   Mods |= SISrcMods::OP_SEL_1;
2772 
2773   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2774   return true;
2775 }
2776 
2777 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2778                                          SDValue &SrcMods) const {
2779   Src = In;
2780   // FIXME: Handle op_sel
2781   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2782   return true;
2783 }
2784 
2785 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2786                                              SDValue &SrcMods) const {
2787   // FIXME: Handle op_sel
2788   return SelectVOP3Mods(In, Src, SrcMods);
2789 }
2790 
2791 // The return value is not whether the match is possible (which it always is),
2792 // but whether or not it a conversion is really used.
2793 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2794                                                    unsigned &Mods) const {
2795   Mods = 0;
2796   SelectVOP3ModsImpl(In, Src, Mods);
2797 
2798   if (Src.getOpcode() == ISD::FP_EXTEND) {
2799     Src = Src.getOperand(0);
2800     assert(Src.getValueType() == MVT::f16);
2801     Src = stripBitcast(Src);
2802 
2803     // Be careful about folding modifiers if we already have an abs. fneg is
2804     // applied last, so we don't want to apply an earlier fneg.
2805     if ((Mods & SISrcMods::ABS) == 0) {
2806       unsigned ModsTmp;
2807       SelectVOP3ModsImpl(Src, Src, ModsTmp);
2808 
2809       if ((ModsTmp & SISrcMods::NEG) != 0)
2810         Mods ^= SISrcMods::NEG;
2811 
2812       if ((ModsTmp & SISrcMods::ABS) != 0)
2813         Mods |= SISrcMods::ABS;
2814     }
2815 
2816     // op_sel/op_sel_hi decide the source type and source.
2817     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2818     // If the sources's op_sel is set, it picks the high half of the source
2819     // register.
2820 
2821     Mods |= SISrcMods::OP_SEL_1;
2822     if (isExtractHiElt(Src, Src)) {
2823       Mods |= SISrcMods::OP_SEL_0;
2824 
2825       // TODO: Should we try to look for neg/abs here?
2826     }
2827 
2828     return true;
2829   }
2830 
2831   return false;
2832 }
2833 
2834 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2835                                                SDValue &SrcMods) const {
2836   unsigned Mods = 0;
2837   SelectVOP3PMadMixModsImpl(In, Src, Mods);
2838   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2839   return true;
2840 }
2841 
2842 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2843   if (In.isUndef())
2844     return CurDAG->getUNDEF(MVT::i32);
2845 
2846   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2847     SDLoc SL(In);
2848     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2849   }
2850 
2851   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2852     SDLoc SL(In);
2853     return CurDAG->getConstant(
2854       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2855   }
2856 
2857   SDValue Src;
2858   if (isExtractHiElt(In, Src))
2859     return Src;
2860 
2861   return SDValue();
2862 }
2863 
2864 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2865   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2866 
2867   const SIRegisterInfo *SIRI =
2868     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2869   const SIInstrInfo * SII =
2870     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2871 
2872   unsigned Limit = 0;
2873   bool AllUsesAcceptSReg = true;
2874   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2875     Limit < 10 && U != E; ++U, ++Limit) {
2876     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2877 
2878     // If the register class is unknown, it could be an unknown
2879     // register class that needs to be an SGPR, e.g. an inline asm
2880     // constraint
2881     if (!RC || SIRI->isSGPRClass(RC))
2882       return false;
2883 
2884     if (RC != &AMDGPU::VS_32RegClass) {
2885       AllUsesAcceptSReg = false;
2886       SDNode * User = *U;
2887       if (User->isMachineOpcode()) {
2888         unsigned Opc = User->getMachineOpcode();
2889         MCInstrDesc Desc = SII->get(Opc);
2890         if (Desc.isCommutable()) {
2891           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2892           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2893           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2894             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2895             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2896             if (CommutedRC == &AMDGPU::VS_32RegClass)
2897               AllUsesAcceptSReg = true;
2898           }
2899         }
2900       }
2901       // If "AllUsesAcceptSReg == false" so far we haven't suceeded
2902       // commuting current user. This means have at least one use
2903       // that strictly require VGPR. Thus, we will not attempt to commute
2904       // other user instructions.
2905       if (!AllUsesAcceptSReg)
2906         break;
2907     }
2908   }
2909   return !AllUsesAcceptSReg && (Limit < 10);
2910 }
2911 
2912 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2913   auto Ld = cast<LoadSDNode>(N);
2914 
2915   return Ld->getAlignment() >= 4 &&
2916         (
2917           (
2918             (
2919               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS       ||
2920               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT
2921             )
2922             &&
2923             !N->isDivergent()
2924           )
2925           ||
2926           (
2927             Subtarget->getScalarizeGlobalBehavior() &&
2928             Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2929             Ld->isSimple() &&
2930             !N->isDivergent() &&
2931             static_cast<const SITargetLowering *>(
2932               getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)
2933           )
2934         );
2935 }
2936 
2937 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2938   const AMDGPUTargetLowering& Lowering =
2939     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2940   bool IsModified = false;
2941   do {
2942     IsModified = false;
2943 
2944     // Go over all selected nodes and try to fold them a bit more
2945     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2946     while (Position != CurDAG->allnodes_end()) {
2947       SDNode *Node = &*Position++;
2948       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
2949       if (!MachineNode)
2950         continue;
2951 
2952       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
2953       if (ResNode != Node) {
2954         if (ResNode)
2955           ReplaceUses(Node, ResNode);
2956         IsModified = true;
2957       }
2958     }
2959     CurDAG->RemoveDeadNodes();
2960   } while (IsModified);
2961 }
2962 
2963 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
2964   Subtarget = &MF.getSubtarget<R600Subtarget>();
2965   return SelectionDAGISel::runOnMachineFunction(MF);
2966 }
2967 
2968 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
2969   if (!N->readMem())
2970     return false;
2971   if (CbId == -1)
2972     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2973            N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
2974 
2975   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
2976 }
2977 
2978 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
2979                                                          SDValue& IntPtr) {
2980   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
2981     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
2982                                        true);
2983     return true;
2984   }
2985   return false;
2986 }
2987 
2988 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
2989     SDValue& BaseReg, SDValue &Offset) {
2990   if (!isa<ConstantSDNode>(Addr)) {
2991     BaseReg = Addr;
2992     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
2993     return true;
2994   }
2995   return false;
2996 }
2997 
2998 void R600DAGToDAGISel::Select(SDNode *N) {
2999   unsigned int Opc = N->getOpcode();
3000   if (N->isMachineOpcode()) {
3001     N->setNodeId(-1);
3002     return;   // Already selected.
3003   }
3004 
3005   switch (Opc) {
3006   default: break;
3007   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
3008   case ISD::SCALAR_TO_VECTOR:
3009   case ISD::BUILD_VECTOR: {
3010     EVT VT = N->getValueType(0);
3011     unsigned NumVectorElts = VT.getVectorNumElements();
3012     unsigned RegClassID;
3013     // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
3014     // that adds a 128 bits reg copy when going through TwoAddressInstructions
3015     // pass. We want to avoid 128 bits copies as much as possible because they
3016     // can't be bundled by our scheduler.
3017     switch(NumVectorElts) {
3018     case 2: RegClassID = R600::R600_Reg64RegClassID; break;
3019     case 4:
3020       if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
3021         RegClassID = R600::R600_Reg128VerticalRegClassID;
3022       else
3023         RegClassID = R600::R600_Reg128RegClassID;
3024       break;
3025     default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
3026     }
3027     SelectBuildVector(N, RegClassID);
3028     return;
3029   }
3030   }
3031 
3032   SelectCode(N);
3033 }
3034 
3035 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
3036                                           SDValue &Offset) {
3037   ConstantSDNode *C;
3038   SDLoc DL(Addr);
3039 
3040   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
3041     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3042     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3043   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
3044              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
3045     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3046     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3047   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
3048             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
3049     Base = Addr.getOperand(0);
3050     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3051   } else {
3052     Base = Addr;
3053     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
3054   }
3055 
3056   return true;
3057 }
3058 
3059 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
3060                                           SDValue &Offset) {
3061   ConstantSDNode *IMMOffset;
3062 
3063   if (Addr.getOpcode() == ISD::ADD
3064       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
3065       && isInt<16>(IMMOffset->getZExtValue())) {
3066 
3067       Base = Addr.getOperand(0);
3068       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3069                                          MVT::i32);
3070       return true;
3071   // If the pointer address is constant, we can move it to the offset field.
3072   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
3073              && isInt<16>(IMMOffset->getZExtValue())) {
3074     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
3075                                   SDLoc(CurDAG->getEntryNode()),
3076                                   R600::ZERO, MVT::i32);
3077     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3078                                        MVT::i32);
3079     return true;
3080   }
3081 
3082   // Default case, no offset
3083   Base = Addr;
3084   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
3085   return true;
3086 }
3087