1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPUArgumentUsageInfo.h"
16 #include "AMDGPUISelLowering.h" // For AMDGPUISD
17 #include "AMDGPUInstrInfo.h"
18 #include "AMDGPUPerfHintAnalysis.h"
19 #include "AMDGPUSubtarget.h"
20 #include "AMDGPUTargetMachine.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "SIDefines.h"
23 #include "SIISelLowering.h"
24 #include "SIInstrInfo.h"
25 #include "SIMachineFunctionInfo.h"
26 #include "SIRegisterInfo.h"
27 #include "llvm/ADT/APInt.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
31 #include "llvm/Analysis/LoopInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/FunctionLoweringInfo.h"
34 #include "llvm/CodeGen/ISDOpcodes.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/SelectionDAG.h"
38 #include "llvm/CodeGen/SelectionDAGISel.h"
39 #include "llvm/CodeGen/SelectionDAGNodes.h"
40 #include "llvm/CodeGen/ValueTypes.h"
41 #include "llvm/IR/BasicBlock.h"
42 #include "llvm/InitializePasses.h"
43 #ifdef EXPENSIVE_CHECKS
44 #include "llvm/IR/Dominators.h"
45 #endif
46 #include "llvm/IR/Instruction.h"
47 #include "llvm/MC/MCInstrDesc.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CodeGen.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/MachineValueType.h"
52 #include "llvm/Support/MathExtras.h"
53 #include <cassert>
54 #include <cstdint>
55 #include <new>
56 #include <vector>
57 
58 #define DEBUG_TYPE "isel"
59 
60 using namespace llvm;
61 
62 namespace llvm {
63 
64 class R600InstrInfo;
65 
66 } // end namespace llvm
67 
68 //===----------------------------------------------------------------------===//
69 // Instruction Selector Implementation
70 //===----------------------------------------------------------------------===//
71 
72 namespace {
73 
74 static bool isNullConstantOrUndef(SDValue V) {
75   if (V.isUndef())
76     return true;
77 
78   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
79   return Const != nullptr && Const->isNullValue();
80 }
81 
82 static bool getConstantValue(SDValue N, uint32_t &Out) {
83   // This is only used for packed vectors, where ussing 0 for undef should
84   // always be good.
85   if (N.isUndef()) {
86     Out = 0;
87     return true;
88   }
89 
90   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
91     Out = C->getAPIntValue().getSExtValue();
92     return true;
93   }
94 
95   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
96     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
97     return true;
98   }
99 
100   return false;
101 }
102 
103 // TODO: Handle undef as zero
104 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
105                                  bool Negate = false) {
106   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
107   uint32_t LHSVal, RHSVal;
108   if (getConstantValue(N->getOperand(0), LHSVal) &&
109       getConstantValue(N->getOperand(1), RHSVal)) {
110     SDLoc SL(N);
111     uint32_t K = Negate ?
112       (-LHSVal & 0xffff) | (-RHSVal << 16) :
113       (LHSVal & 0xffff) | (RHSVal << 16);
114     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
115                               DAG.getTargetConstant(K, SL, MVT::i32));
116   }
117 
118   return nullptr;
119 }
120 
121 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
122   return packConstantV2I16(N, DAG, true);
123 }
124 
125 /// AMDGPU specific code to select AMDGPU machine instructions for
126 /// SelectionDAG operations.
127 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
128   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
129   // make the right decision when generating code for different targets.
130   const GCNSubtarget *Subtarget;
131 
132   // Default FP mode for the current function.
133   AMDGPU::SIModeRegisterDefaults Mode;
134 
135   bool EnableLateStructurizeCFG;
136 
137 public:
138   explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
139                               CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
140     : SelectionDAGISel(*TM, OptLevel) {
141     EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
142   }
143   ~AMDGPUDAGToDAGISel() override = default;
144 
145   void getAnalysisUsage(AnalysisUsage &AU) const override {
146     AU.addRequired<AMDGPUArgumentUsageInfo>();
147     AU.addRequired<LegacyDivergenceAnalysis>();
148 #ifdef EXPENSIVE_CHECKS
149     AU.addRequired<DominatorTreeWrapperPass>();
150     AU.addRequired<LoopInfoWrapperPass>();
151 #endif
152     SelectionDAGISel::getAnalysisUsage(AU);
153   }
154 
155   bool matchLoadD16FromBuildVector(SDNode *N) const;
156 
157   bool runOnMachineFunction(MachineFunction &MF) override;
158   void PreprocessISelDAG() override;
159   void Select(SDNode *N) override;
160   StringRef getPassName() const override;
161   void PostprocessISelDAG() override;
162 
163 protected:
164   void SelectBuildVector(SDNode *N, unsigned RegClassID);
165 
166 private:
167   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
168   bool isNoNanSrc(SDValue N) const;
169   bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
170   bool isNegInlineImmediate(const SDNode *N) const {
171     return isInlineImmediate(N, true);
172   }
173 
174   bool isInlineImmediate16(int64_t Imm) const {
175     return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());
176   }
177 
178   bool isInlineImmediate32(int64_t Imm) const {
179     return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());
180   }
181 
182   bool isInlineImmediate64(int64_t Imm) const {
183     return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());
184   }
185 
186   bool isInlineImmediate(const APFloat &Imm) const {
187     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
188   }
189 
190   bool isVGPRImm(const SDNode *N) const;
191   bool isUniformLoad(const SDNode *N) const;
192   bool isUniformBr(const SDNode *N) const;
193 
194   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
195 
196   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
197   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
198   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
199 
200   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
201   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
202   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
203   bool isDSOffsetLegal(SDValue Base, unsigned Offset,
204                        unsigned OffsetBits) const;
205   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
206   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
207                                  SDValue &Offset1) const;
208   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
209                                   SDValue &Offset1) const;
210   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
211                           SDValue &Offset1, bool IsDS128) const;
212   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
213                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
214                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
215                    SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
216   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
217                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
218                          SDValue &SLC, SDValue &TFE, SDValue &DLC,
219                          SDValue &SWZ) const;
220   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
221                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
222                          SDValue &SLC) const;
223   bool SelectMUBUFScratchOffen(SDNode *Parent,
224                                SDValue Addr, SDValue &RSrc, SDValue &VAddr,
225                                SDValue &SOffset, SDValue &ImmOffset) const;
226   bool SelectMUBUFScratchOffset(SDNode *Parent,
227                                 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
228                                 SDValue &Offset) const;
229 
230   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
231                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
232                          SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
233   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
234                          SDValue &Offset, SDValue &SLC) const;
235   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
236                          SDValue &Offset) const;
237 
238   template <bool IsSigned>
239   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
240                         SDValue &Offset) const;
241   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
242                          SDValue &VOffset, SDValue &Offset) const;
243   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
244                           SDValue &Offset) const;
245 
246   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
247                         bool &Imm) const;
248   SDValue Expand32BitAddress(SDValue Addr) const;
249   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
250                   bool &Imm) const;
251   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
252   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
253   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
254   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
255   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
256   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
257 
258   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
259   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
260   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
261   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
262   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
263                        SDValue &Clamp, SDValue &Omod) const;
264   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
265                          SDValue &Clamp, SDValue &Omod) const;
266 
267   bool SelectVOP3OMods(SDValue In, SDValue &Src,
268                        SDValue &Clamp, SDValue &Omod) const;
269 
270   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
271 
272   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
273 
274   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
275   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
276   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
277 
278   SDValue getHi16Elt(SDValue In) const;
279 
280   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
281 
282   void SelectADD_SUB_I64(SDNode *N);
283   void SelectAddcSubb(SDNode *N);
284   void SelectUADDO_USUBO(SDNode *N);
285   void SelectDIV_SCALE(SDNode *N);
286   void SelectMAD_64_32(SDNode *N);
287   void SelectFMA_W_CHAIN(SDNode *N);
288   void SelectFMUL_W_CHAIN(SDNode *N);
289 
290   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
291                    uint32_t Offset, uint32_t Width);
292   void SelectS_BFEFromShifts(SDNode *N);
293   void SelectS_BFE(SDNode *N);
294   bool isCBranchSCC(const SDNode *N) const;
295   void SelectBRCOND(SDNode *N);
296   void SelectFMAD_FMA(SDNode *N);
297   void SelectATOMIC_CMP_SWAP(SDNode *N);
298   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
299   void SelectDS_GWS(SDNode *N, unsigned IntrID);
300   void SelectInterpP1F16(SDNode *N);
301   void SelectINTRINSIC_W_CHAIN(SDNode *N);
302   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
303   void SelectINTRINSIC_VOID(SDNode *N);
304 
305 protected:
306   // Include the pieces autogenerated from the target description.
307 #include "AMDGPUGenDAGISel.inc"
308 };
309 
310 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
311   const R600Subtarget *Subtarget;
312 
313   bool isConstantLoad(const MemSDNode *N, int cbID) const;
314   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
315   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
316                                        SDValue& Offset);
317 public:
318   explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
319       AMDGPUDAGToDAGISel(TM, OptLevel) {}
320 
321   void Select(SDNode *N) override;
322 
323   bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
324                           SDValue &Offset) override;
325   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
326                           SDValue &Offset) override;
327 
328   bool runOnMachineFunction(MachineFunction &MF) override;
329 
330   void PreprocessISelDAG() override {}
331 
332 protected:
333   // Include the pieces autogenerated from the target description.
334 #include "R600GenDAGISel.inc"
335 };
336 
337 static SDValue stripBitcast(SDValue Val) {
338   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
339 }
340 
341 // Figure out if this is really an extract of the high 16-bits of a dword.
342 static bool isExtractHiElt(SDValue In, SDValue &Out) {
343   In = stripBitcast(In);
344   if (In.getOpcode() != ISD::TRUNCATE)
345     return false;
346 
347   SDValue Srl = In.getOperand(0);
348   if (Srl.getOpcode() == ISD::SRL) {
349     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
350       if (ShiftAmt->getZExtValue() == 16) {
351         Out = stripBitcast(Srl.getOperand(0));
352         return true;
353       }
354     }
355   }
356 
357   return false;
358 }
359 
360 // Look through operations that obscure just looking at the low 16-bits of the
361 // same register.
362 static SDValue stripExtractLoElt(SDValue In) {
363   if (In.getOpcode() == ISD::TRUNCATE) {
364     SDValue Src = In.getOperand(0);
365     if (Src.getValueType().getSizeInBits() == 32)
366       return stripBitcast(Src);
367   }
368 
369   return In;
370 }
371 
372 }  // end anonymous namespace
373 
374 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
375                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
376 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
377 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
378 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
379 #ifdef EXPENSIVE_CHECKS
380 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
381 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
382 #endif
383 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
384                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
385 
386 /// This pass converts a legalized DAG into a AMDGPU-specific
387 // DAG, ready for instruction scheduling.
388 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
389                                         CodeGenOpt::Level OptLevel) {
390   return new AMDGPUDAGToDAGISel(TM, OptLevel);
391 }
392 
393 /// This pass converts a legalized DAG into a R600-specific
394 // DAG, ready for instruction scheduling.
395 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
396                                       CodeGenOpt::Level OptLevel) {
397   return new R600DAGToDAGISel(TM, OptLevel);
398 }
399 
400 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
401 #ifdef EXPENSIVE_CHECKS
402   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
403   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
404   for (auto &L : LI->getLoopsInPreorder()) {
405     assert(L->isLCSSAForm(DT));
406   }
407 #endif
408   Subtarget = &MF.getSubtarget<GCNSubtarget>();
409   Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
410   return SelectionDAGISel::runOnMachineFunction(MF);
411 }
412 
413 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
414   assert(Subtarget->d16PreservesUnusedBits());
415   MVT VT = N->getValueType(0).getSimpleVT();
416   if (VT != MVT::v2i16 && VT != MVT::v2f16)
417     return false;
418 
419   SDValue Lo = N->getOperand(0);
420   SDValue Hi = N->getOperand(1);
421 
422   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
423 
424   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
425   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
426   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
427 
428   // Need to check for possible indirect dependencies on the other half of the
429   // vector to avoid introducing a cycle.
430   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
431     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
432 
433     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
434     SDValue Ops[] = {
435       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
436     };
437 
438     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
439     if (LdHi->getMemoryVT() == MVT::i8) {
440       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
441         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
442     } else {
443       assert(LdHi->getMemoryVT() == MVT::i16);
444     }
445 
446     SDValue NewLoadHi =
447       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
448                                   Ops, LdHi->getMemoryVT(),
449                                   LdHi->getMemOperand());
450 
451     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
452     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
453     return true;
454   }
455 
456   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
457   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
458   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
459   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
460   if (LdLo && Lo.hasOneUse()) {
461     SDValue TiedIn = getHi16Elt(Hi);
462     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
463       return false;
464 
465     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
466     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
467     if (LdLo->getMemoryVT() == MVT::i8) {
468       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
469         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
470     } else {
471       assert(LdLo->getMemoryVT() == MVT::i16);
472     }
473 
474     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
475 
476     SDValue Ops[] = {
477       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
478     };
479 
480     SDValue NewLoadLo =
481       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
482                                   Ops, LdLo->getMemoryVT(),
483                                   LdLo->getMemOperand());
484 
485     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
486     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
487     return true;
488   }
489 
490   return false;
491 }
492 
493 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
494   if (!Subtarget->d16PreservesUnusedBits())
495     return;
496 
497   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
498 
499   bool MadeChange = false;
500   while (Position != CurDAG->allnodes_begin()) {
501     SDNode *N = &*--Position;
502     if (N->use_empty())
503       continue;
504 
505     switch (N->getOpcode()) {
506     case ISD::BUILD_VECTOR:
507       MadeChange |= matchLoadD16FromBuildVector(N);
508       break;
509     default:
510       break;
511     }
512   }
513 
514   if (MadeChange) {
515     CurDAG->RemoveDeadNodes();
516     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
517                CurDAG->dump(););
518   }
519 }
520 
521 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
522   if (TM.Options.NoNaNsFPMath)
523     return true;
524 
525   // TODO: Move into isKnownNeverNaN
526   if (N->getFlags().hasNoNaNs())
527     return true;
528 
529   return CurDAG->isKnownNeverNaN(N);
530 }
531 
532 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
533                                            bool Negated) const {
534   if (N->isUndef())
535     return true;
536 
537   const SIInstrInfo *TII = Subtarget->getInstrInfo();
538   if (Negated) {
539     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
540       return TII->isInlineConstant(-C->getAPIntValue());
541 
542     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
543       return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
544 
545   } else {
546     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
547       return TII->isInlineConstant(C->getAPIntValue());
548 
549     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
550       return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
551   }
552 
553   return false;
554 }
555 
556 /// Determine the register class for \p OpNo
557 /// \returns The register class of the virtual register that will be used for
558 /// the given operand number \OpNo or NULL if the register class cannot be
559 /// determined.
560 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
561                                                           unsigned OpNo) const {
562   if (!N->isMachineOpcode()) {
563     if (N->getOpcode() == ISD::CopyToReg) {
564       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
565       if (Reg.isVirtual()) {
566         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
567         return MRI.getRegClass(Reg);
568       }
569 
570       const SIRegisterInfo *TRI
571         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
572       return TRI->getPhysRegClass(Reg);
573     }
574 
575     return nullptr;
576   }
577 
578   switch (N->getMachineOpcode()) {
579   default: {
580     const MCInstrDesc &Desc =
581         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
582     unsigned OpIdx = Desc.getNumDefs() + OpNo;
583     if (OpIdx >= Desc.getNumOperands())
584       return nullptr;
585     int RegClass = Desc.OpInfo[OpIdx].RegClass;
586     if (RegClass == -1)
587       return nullptr;
588 
589     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
590   }
591   case AMDGPU::REG_SEQUENCE: {
592     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
593     const TargetRegisterClass *SuperRC =
594         Subtarget->getRegisterInfo()->getRegClass(RCID);
595 
596     SDValue SubRegOp = N->getOperand(OpNo + 1);
597     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
598     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
599                                                               SubRegIdx);
600   }
601   }
602 }
603 
604 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
605                                          SDValue Glue) const {
606   SmallVector <SDValue, 8> Ops;
607   Ops.push_back(NewChain); // Replace the chain.
608   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
609     Ops.push_back(N->getOperand(i));
610 
611   Ops.push_back(Glue);
612   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
613 }
614 
615 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
616   const SITargetLowering& Lowering =
617     *static_cast<const SITargetLowering*>(getTargetLowering());
618 
619   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
620 
621   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
622   return glueCopyToOp(N, M0, M0.getValue(1));
623 }
624 
625 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
626   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
627   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
628     if (Subtarget->ldsRequiresM0Init())
629       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
630   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
631     MachineFunction &MF = CurDAG->getMachineFunction();
632     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
633     return
634         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
635   }
636   return N;
637 }
638 
639 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
640                                                   EVT VT) const {
641   SDNode *Lo = CurDAG->getMachineNode(
642       AMDGPU::S_MOV_B32, DL, MVT::i32,
643       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
644   SDNode *Hi =
645       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
646                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
647   const SDValue Ops[] = {
648       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
649       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
650       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
651 
652   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
653 }
654 
655 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
656   EVT VT = N->getValueType(0);
657   unsigned NumVectorElts = VT.getVectorNumElements();
658   EVT EltVT = VT.getVectorElementType();
659   SDLoc DL(N);
660   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
661 
662   if (NumVectorElts == 1) {
663     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
664                          RegClass);
665     return;
666   }
667 
668   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
669                                   "supported yet");
670   // 32 = Max Num Vector Elements
671   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
672   // 1 = Vector Register Class
673   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
674 
675   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
676                Triple::amdgcn;
677   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
678   bool IsRegSeq = true;
679   unsigned NOps = N->getNumOperands();
680   for (unsigned i = 0; i < NOps; i++) {
681     // XXX: Why is this here?
682     if (isa<RegisterSDNode>(N->getOperand(i))) {
683       IsRegSeq = false;
684       break;
685     }
686     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
687                          : R600RegisterInfo::getSubRegFromChannel(i);
688     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
689     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
690   }
691   if (NOps != NumVectorElts) {
692     // Fill in the missing undef elements if this was a scalar_to_vector.
693     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
694     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
695                                                    DL, EltVT);
696     for (unsigned i = NOps; i < NumVectorElts; ++i) {
697       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
698                            : R600RegisterInfo::getSubRegFromChannel(i);
699       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
700       RegSeqArgs[1 + (2 * i) + 1] =
701           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
702     }
703   }
704 
705   if (!IsRegSeq)
706     SelectCode(N);
707   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
708 }
709 
710 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
711   unsigned int Opc = N->getOpcode();
712   if (N->isMachineOpcode()) {
713     N->setNodeId(-1);
714     return;   // Already selected.
715   }
716 
717   // isa<MemSDNode> almost works but is slightly too permissive for some DS
718   // intrinsics.
719   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
720       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
721        Opc == ISD::ATOMIC_LOAD_FADD ||
722        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
723        Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
724     N = glueCopyToM0LDSInit(N);
725     SelectCode(N);
726     return;
727   }
728 
729   switch (Opc) {
730   default:
731     break;
732   // We are selecting i64 ADD here instead of custom lower it during
733   // DAG legalization, so we can fold some i64 ADDs used for address
734   // calculation into the LOAD and STORE instructions.
735   case ISD::ADDC:
736   case ISD::ADDE:
737   case ISD::SUBC:
738   case ISD::SUBE: {
739     if (N->getValueType(0) != MVT::i64)
740       break;
741 
742     SelectADD_SUB_I64(N);
743     return;
744   }
745   case ISD::ADDCARRY:
746   case ISD::SUBCARRY:
747     if (N->getValueType(0) != MVT::i32)
748       break;
749 
750     SelectAddcSubb(N);
751     return;
752   case ISD::UADDO:
753   case ISD::USUBO: {
754     SelectUADDO_USUBO(N);
755     return;
756   }
757   case AMDGPUISD::FMUL_W_CHAIN: {
758     SelectFMUL_W_CHAIN(N);
759     return;
760   }
761   case AMDGPUISD::FMA_W_CHAIN: {
762     SelectFMA_W_CHAIN(N);
763     return;
764   }
765 
766   case ISD::SCALAR_TO_VECTOR:
767   case ISD::BUILD_VECTOR: {
768     EVT VT = N->getValueType(0);
769     unsigned NumVectorElts = VT.getVectorNumElements();
770     if (VT.getScalarSizeInBits() == 16) {
771       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
772         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
773           ReplaceNode(N, Packed);
774           return;
775         }
776       }
777 
778       break;
779     }
780 
781     assert(VT.getVectorElementType().bitsEq(MVT::i32));
782     unsigned RegClassID =
783         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
784     SelectBuildVector(N, RegClassID);
785     return;
786   }
787   case ISD::BUILD_PAIR: {
788     SDValue RC, SubReg0, SubReg1;
789     SDLoc DL(N);
790     if (N->getValueType(0) == MVT::i128) {
791       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
792       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
793       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
794     } else if (N->getValueType(0) == MVT::i64) {
795       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
796       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
797       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
798     } else {
799       llvm_unreachable("Unhandled value type for BUILD_PAIR");
800     }
801     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
802                             N->getOperand(1), SubReg1 };
803     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
804                                           N->getValueType(0), Ops));
805     return;
806   }
807 
808   case ISD::Constant:
809   case ISD::ConstantFP: {
810     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
811       break;
812 
813     uint64_t Imm;
814     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
815       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
816     else {
817       ConstantSDNode *C = cast<ConstantSDNode>(N);
818       Imm = C->getZExtValue();
819     }
820 
821     SDLoc DL(N);
822     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
823     return;
824   }
825   case AMDGPUISD::BFE_I32:
826   case AMDGPUISD::BFE_U32: {
827     // There is a scalar version available, but unlike the vector version which
828     // has a separate operand for the offset and width, the scalar version packs
829     // the width and offset into a single operand. Try to move to the scalar
830     // version if the offsets are constant, so that we can try to keep extended
831     // loads of kernel arguments in SGPRs.
832 
833     // TODO: Technically we could try to pattern match scalar bitshifts of
834     // dynamic values, but it's probably not useful.
835     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
836     if (!Offset)
837       break;
838 
839     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
840     if (!Width)
841       break;
842 
843     bool Signed = Opc == AMDGPUISD::BFE_I32;
844 
845     uint32_t OffsetVal = Offset->getZExtValue();
846     uint32_t WidthVal = Width->getZExtValue();
847 
848     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
849                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
850     return;
851   }
852   case AMDGPUISD::DIV_SCALE: {
853     SelectDIV_SCALE(N);
854     return;
855   }
856   case AMDGPUISD::MAD_I64_I32:
857   case AMDGPUISD::MAD_U64_U32: {
858     SelectMAD_64_32(N);
859     return;
860   }
861   case ISD::CopyToReg: {
862     const SITargetLowering& Lowering =
863       *static_cast<const SITargetLowering*>(getTargetLowering());
864     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
865     break;
866   }
867   case ISD::AND:
868   case ISD::SRL:
869   case ISD::SRA:
870   case ISD::SIGN_EXTEND_INREG:
871     if (N->getValueType(0) != MVT::i32)
872       break;
873 
874     SelectS_BFE(N);
875     return;
876   case ISD::BRCOND:
877     SelectBRCOND(N);
878     return;
879   case ISD::FMAD:
880   case ISD::FMA:
881     SelectFMAD_FMA(N);
882     return;
883   case AMDGPUISD::ATOMIC_CMP_SWAP:
884     SelectATOMIC_CMP_SWAP(N);
885     return;
886   case AMDGPUISD::CVT_PKRTZ_F16_F32:
887   case AMDGPUISD::CVT_PKNORM_I16_F32:
888   case AMDGPUISD::CVT_PKNORM_U16_F32:
889   case AMDGPUISD::CVT_PK_U16_U32:
890   case AMDGPUISD::CVT_PK_I16_I32: {
891     // Hack around using a legal type if f16 is illegal.
892     if (N->getValueType(0) == MVT::i32) {
893       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
894       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
895                               { N->getOperand(0), N->getOperand(1) });
896       SelectCode(N);
897       return;
898     }
899 
900     break;
901   }
902   case ISD::INTRINSIC_W_CHAIN: {
903     SelectINTRINSIC_W_CHAIN(N);
904     return;
905   }
906   case ISD::INTRINSIC_WO_CHAIN: {
907     SelectINTRINSIC_WO_CHAIN(N);
908     return;
909   }
910   case ISD::INTRINSIC_VOID: {
911     SelectINTRINSIC_VOID(N);
912     return;
913   }
914   }
915 
916   SelectCode(N);
917 }
918 
919 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
920   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
921   const Instruction *Term = BB->getTerminator();
922   return Term->getMetadata("amdgpu.uniform") ||
923          Term->getMetadata("structurizecfg.uniform");
924 }
925 
926 StringRef AMDGPUDAGToDAGISel::getPassName() const {
927   return "AMDGPU DAG->DAG Pattern Instruction Selection";
928 }
929 
930 //===----------------------------------------------------------------------===//
931 // Complex Patterns
932 //===----------------------------------------------------------------------===//
933 
934 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
935                                             SDValue &Offset) {
936   return false;
937 }
938 
939 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
940                                             SDValue &Offset) {
941   ConstantSDNode *C;
942   SDLoc DL(Addr);
943 
944   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
945     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
946     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
947   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
948              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
949     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
950     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
951   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
952             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
953     Base = Addr.getOperand(0);
954     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
955   } else {
956     Base = Addr;
957     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
958   }
959 
960   return true;
961 }
962 
963 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
964                                                        const SDLoc &DL) const {
965   SDNode *Mov = CurDAG->getMachineNode(
966     AMDGPU::S_MOV_B32, DL, MVT::i32,
967     CurDAG->getTargetConstant(Val, DL, MVT::i32));
968   return SDValue(Mov, 0);
969 }
970 
971 // FIXME: Should only handle addcarry/subcarry
972 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
973   SDLoc DL(N);
974   SDValue LHS = N->getOperand(0);
975   SDValue RHS = N->getOperand(1);
976 
977   unsigned Opcode = N->getOpcode();
978   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
979   bool ProduceCarry =
980       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
981   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
982 
983   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
984   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
985 
986   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
987                                        DL, MVT::i32, LHS, Sub0);
988   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
989                                        DL, MVT::i32, LHS, Sub1);
990 
991   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
992                                        DL, MVT::i32, RHS, Sub0);
993   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
994                                        DL, MVT::i32, RHS, Sub1);
995 
996   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
997 
998   static const unsigned OpcMap[2][2][2] = {
999       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
1000        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
1001       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
1002        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
1003 
1004   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
1005   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
1006 
1007   SDNode *AddLo;
1008   if (!ConsumeCarry) {
1009     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1010     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
1011   } else {
1012     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1013     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1014   }
1015   SDValue AddHiArgs[] = {
1016     SDValue(Hi0, 0),
1017     SDValue(Hi1, 0),
1018     SDValue(AddLo, 1)
1019   };
1020   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
1021 
1022   SDValue RegSequenceArgs[] = {
1023     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1024     SDValue(AddLo,0),
1025     Sub0,
1026     SDValue(AddHi,0),
1027     Sub1,
1028   };
1029   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1030                                                MVT::i64, RegSequenceArgs);
1031 
1032   if (ProduceCarry) {
1033     // Replace the carry-use
1034     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1035   }
1036 
1037   // Replace the remaining uses.
1038   ReplaceNode(N, RegSequence);
1039 }
1040 
1041 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1042   SDLoc DL(N);
1043   SDValue LHS = N->getOperand(0);
1044   SDValue RHS = N->getOperand(1);
1045   SDValue CI = N->getOperand(2);
1046 
1047   if (N->isDivergent()) {
1048     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
1049                                                    : AMDGPU::V_SUBB_U32_e64;
1050     CurDAG->SelectNodeTo(
1051         N, Opc, N->getVTList(),
1052         {LHS, RHS, CI,
1053          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1054   } else {
1055     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
1056                                                    : AMDGPU::S_SUB_CO_PSEUDO;
1057     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
1058   }
1059 }
1060 
1061 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1062   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1063   // carry out despite the _i32 name. These were renamed in VI to _U32.
1064   // FIXME: We should probably rename the opcodes here.
1065   bool IsAdd = N->getOpcode() == ISD::UADDO;
1066   bool IsVALU = N->isDivergent();
1067 
1068   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
1069        ++UI)
1070     if (UI.getUse().getResNo() == 1) {
1071       if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
1072           (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
1073         IsVALU = true;
1074         break;
1075       }
1076     }
1077 
1078   if (IsVALU) {
1079     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1080 
1081     CurDAG->SelectNodeTo(
1082         N, Opc, N->getVTList(),
1083         {N->getOperand(0), N->getOperand(1),
1084          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1085   } else {
1086     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
1087                                                 : AMDGPU::S_USUBO_PSEUDO;
1088 
1089     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
1090                          {N->getOperand(0), N->getOperand(1)});
1091   }
1092 }
1093 
1094 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1095   SDLoc SL(N);
1096   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1097   SDValue Ops[10];
1098 
1099   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1100   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1101   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1102   Ops[8] = N->getOperand(0);
1103   Ops[9] = N->getOperand(4);
1104 
1105   CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops);
1106 }
1107 
1108 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1109   SDLoc SL(N);
1110   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
1111   SDValue Ops[8];
1112 
1113   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1114   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1115   Ops[6] = N->getOperand(0);
1116   Ops[7] = N->getOperand(3);
1117 
1118   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1119 }
1120 
1121 // We need to handle this here because tablegen doesn't support matching
1122 // instructions with multiple outputs.
1123 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1124   SDLoc SL(N);
1125   EVT VT = N->getValueType(0);
1126 
1127   assert(VT == MVT::f32 || VT == MVT::f64);
1128 
1129   unsigned Opc
1130     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
1131 
1132   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
1133   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1134 }
1135 
1136 // We need to handle this here because tablegen doesn't support matching
1137 // instructions with multiple outputs.
1138 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1139   SDLoc SL(N);
1140   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1141   unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32;
1142 
1143   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1144   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1145                     Clamp };
1146   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1147 }
1148 
1149 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset,
1150                                          unsigned OffsetBits) const {
1151   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
1152       (OffsetBits == 8 && !isUInt<8>(Offset)))
1153     return false;
1154 
1155   if (Subtarget->hasUsableDSOffset() ||
1156       Subtarget->unsafeDSOffsetFoldingEnabled())
1157     return true;
1158 
1159   // On Southern Islands instruction with a negative base value and an offset
1160   // don't seem to work.
1161   return CurDAG->SignBitIsZero(Base);
1162 }
1163 
1164 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1165                                               SDValue &Offset) const {
1166   SDLoc DL(Addr);
1167   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1168     SDValue N0 = Addr.getOperand(0);
1169     SDValue N1 = Addr.getOperand(1);
1170     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1171     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
1172       // (add n0, c0)
1173       Base = N0;
1174       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1175       return true;
1176     }
1177   } else if (Addr.getOpcode() == ISD::SUB) {
1178     // sub C, x -> add (sub 0, x), C
1179     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1180       int64_t ByteOffset = C->getSExtValue();
1181       if (isUInt<16>(ByteOffset)) {
1182         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1183 
1184         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1185         // the known bits in isDSOffsetLegal. We need to emit the selected node
1186         // here, so this is thrown away.
1187         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1188                                       Zero, Addr.getOperand(1));
1189 
1190         if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
1191           SmallVector<SDValue, 3> Opnds;
1192           Opnds.push_back(Zero);
1193           Opnds.push_back(Addr.getOperand(1));
1194 
1195           // FIXME: Select to VOP3 version for with-carry.
1196           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1197           if (Subtarget->hasAddNoCarry()) {
1198             SubOp = AMDGPU::V_SUB_U32_e64;
1199             Opnds.push_back(
1200                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1201           }
1202 
1203           MachineSDNode *MachineSub =
1204               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1205 
1206           Base = SDValue(MachineSub, 0);
1207           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1208           return true;
1209         }
1210       }
1211     }
1212   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1213     // If we have a constant address, prefer to put the constant into the
1214     // offset. This can save moves to load the constant address since multiple
1215     // operations can share the zero base address register, and enables merging
1216     // into read2 / write2 instructions.
1217 
1218     SDLoc DL(Addr);
1219 
1220     if (isUInt<16>(CAddr->getZExtValue())) {
1221       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1222       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1223                                  DL, MVT::i32, Zero);
1224       Base = SDValue(MovZero, 0);
1225       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1226       return true;
1227     }
1228   }
1229 
1230   // default case
1231   Base = Addr;
1232   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1233   return true;
1234 }
1235 
1236 // TODO: If offset is too big, put low 16-bit into offset.
1237 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1238                                                    SDValue &Offset0,
1239                                                    SDValue &Offset1) const {
1240   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, false);
1241 }
1242 
1243 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1244                                                     SDValue &Offset0,
1245                                                     SDValue &Offset1) const {
1246   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, true);
1247 }
1248 
1249 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1250                                             SDValue &Offset0, SDValue &Offset1,
1251                                             bool IsDS128) const {
1252   SDLoc DL(Addr);
1253   unsigned Align = IsDS128 ? 8 : 4;
1254 
1255   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1256     SDValue N0 = Addr.getOperand(0);
1257     SDValue N1 = Addr.getOperand(1);
1258     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1259     unsigned OffsetValue0 = C1->getZExtValue() / Align;
1260     unsigned OffsetValue1 = OffsetValue0 + 1;
1261     // (add n0, c0)
1262     if (isDSOffsetLegal(N0, OffsetValue1, 8)) {
1263       Base = N0;
1264       Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1265       Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1266       return true;
1267     }
1268   } else if (Addr.getOpcode() == ISD::SUB) {
1269     // sub C, x -> add (sub 0, x), C
1270     if (const ConstantSDNode *C =
1271             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1272       unsigned OffsetValue0 = C->getZExtValue() / Align;
1273       unsigned OffsetValue1 = OffsetValue0 + 1;
1274 
1275       if (isUInt<8>(OffsetValue0)) {
1276         SDLoc DL(Addr);
1277         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1278 
1279         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1280         // the known bits in isDSOffsetLegal. We need to emit the selected node
1281         // here, so this is thrown away.
1282         SDValue Sub =
1283             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1284 
1285         if (isDSOffsetLegal(Sub, OffsetValue1, 8)) {
1286           SmallVector<SDValue, 3> Opnds;
1287           Opnds.push_back(Zero);
1288           Opnds.push_back(Addr.getOperand(1));
1289           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1290           if (Subtarget->hasAddNoCarry()) {
1291             SubOp = AMDGPU::V_SUB_U32_e64;
1292             Opnds.push_back(
1293                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1294           }
1295 
1296           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1297               SubOp, DL, (IsDS128 ? MVT::i64 : MVT::i32), Opnds);
1298 
1299           Base = SDValue(MachineSub, 0);
1300           Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1301           Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1302           return true;
1303         }
1304       }
1305     }
1306   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1307     unsigned OffsetValue0 = CAddr->getZExtValue() / Align;
1308     unsigned OffsetValue1 = OffsetValue0 + 1;
1309     bool OffsetIsAligned = Align * OffsetValue0 == CAddr->getZExtValue();
1310 
1311     if (isUInt<8>(OffsetValue0) && isUInt<8>(OffsetValue1) && OffsetIsAligned) {
1312       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1313       MachineSDNode *MovZero =
1314           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1315       Base = SDValue(MovZero, 0);
1316       Offset0 = CurDAG->getTargetConstant(OffsetValue0, DL, MVT::i8);
1317       Offset1 = CurDAG->getTargetConstant(OffsetValue1, DL, MVT::i8);
1318       return true;
1319     }
1320   }
1321 
1322   // default case
1323 
1324   Base = Addr;
1325   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1326   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1327   return true;
1328 }
1329 
1330 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1331                                      SDValue &VAddr, SDValue &SOffset,
1332                                      SDValue &Offset, SDValue &Offen,
1333                                      SDValue &Idxen, SDValue &Addr64,
1334                                      SDValue &GLC, SDValue &SLC,
1335                                      SDValue &TFE, SDValue &DLC,
1336                                      SDValue &SWZ) const {
1337   // Subtarget prefers to use flat instruction
1338   // FIXME: This should be a pattern predicate and not reach here
1339   if (Subtarget->useFlatForGlobal())
1340     return false;
1341 
1342   SDLoc DL(Addr);
1343 
1344   if (!GLC.getNode())
1345     GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1346   if (!SLC.getNode())
1347     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1348   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
1349   DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1350   SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
1351 
1352   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1353   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1354   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1355   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1356 
1357   ConstantSDNode *C1 = nullptr;
1358   SDValue N0 = Addr;
1359   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1360     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1361     if (isUInt<32>(C1->getZExtValue()))
1362       N0 = Addr.getOperand(0);
1363     else
1364       C1 = nullptr;
1365   }
1366 
1367   if (N0.getOpcode() == ISD::ADD) {
1368     // (add N2, N3) -> addr64, or
1369     // (add (add N2, N3), C1) -> addr64
1370     SDValue N2 = N0.getOperand(0);
1371     SDValue N3 = N0.getOperand(1);
1372     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1373 
1374     if (N2->isDivergent()) {
1375       if (N3->isDivergent()) {
1376         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1377         // addr64, and construct the resource from a 0 address.
1378         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1379         VAddr = N0;
1380       } else {
1381         // N2 is divergent, N3 is not.
1382         Ptr = N3;
1383         VAddr = N2;
1384       }
1385     } else {
1386       // N2 is not divergent.
1387       Ptr = N2;
1388       VAddr = N3;
1389     }
1390     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1391   } else if (N0->isDivergent()) {
1392     // N0 is divergent. Use it as the addr64, and construct the resource from a
1393     // 0 address.
1394     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1395     VAddr = N0;
1396     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1397   } else {
1398     // N0 -> offset, or
1399     // (N0 + C1) -> offset
1400     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1401     Ptr = N0;
1402   }
1403 
1404   if (!C1) {
1405     // No offset.
1406     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1407     return true;
1408   }
1409 
1410   if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1411     // Legal offset for instruction.
1412     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1413     return true;
1414   }
1415 
1416   // Illegal offset, store it in soffset.
1417   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1418   SOffset =
1419       SDValue(CurDAG->getMachineNode(
1420                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1421                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1422               0);
1423   return true;
1424 }
1425 
1426 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1427                                            SDValue &VAddr, SDValue &SOffset,
1428                                            SDValue &Offset, SDValue &GLC,
1429                                            SDValue &SLC, SDValue &TFE,
1430                                            SDValue &DLC, SDValue &SWZ) const {
1431   SDValue Ptr, Offen, Idxen, Addr64;
1432 
1433   // addr64 bit was removed for volcanic islands.
1434   // FIXME: This should be a pattern predicate and not reach here
1435   if (!Subtarget->hasAddr64())
1436     return false;
1437 
1438   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1439               GLC, SLC, TFE, DLC, SWZ))
1440     return false;
1441 
1442   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1443   if (C->getSExtValue()) {
1444     SDLoc DL(Addr);
1445 
1446     const SITargetLowering& Lowering =
1447       *static_cast<const SITargetLowering*>(getTargetLowering());
1448 
1449     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1450     return true;
1451   }
1452 
1453   return false;
1454 }
1455 
1456 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1457                                            SDValue &VAddr, SDValue &SOffset,
1458                                            SDValue &Offset,
1459                                            SDValue &SLC) const {
1460   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1461   SDValue GLC, TFE, DLC, SWZ;
1462 
1463   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1464 }
1465 
1466 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1467   auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1468   return PSV && PSV->isStack();
1469 }
1470 
1471 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1472   SDLoc DL(N);
1473 
1474   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1475   SDValue TFI =
1476       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1477 
1478   // We rebase the base address into an absolute stack address and hence
1479   // use constant 0 for soffset.
1480   return std::make_pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1481 }
1482 
1483 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1484                                                  SDValue Addr, SDValue &Rsrc,
1485                                                  SDValue &VAddr, SDValue &SOffset,
1486                                                  SDValue &ImmOffset) const {
1487 
1488   SDLoc DL(Addr);
1489   MachineFunction &MF = CurDAG->getMachineFunction();
1490   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1491 
1492   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1493 
1494   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1495     int64_t Imm = CAddr->getSExtValue();
1496     const int64_t NullPtr =
1497         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1498     // Don't fold null pointer.
1499     if (Imm != NullPtr) {
1500       SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1501       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1502         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1503       VAddr = SDValue(MovHighBits, 0);
1504 
1505       // In a call sequence, stores to the argument stack area are relative to the
1506       // stack pointer.
1507       const MachinePointerInfo &PtrInfo
1508         = cast<MemSDNode>(Parent)->getPointerInfo();
1509       SOffset = isStackPtrRelative(PtrInfo)
1510         ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1511         : CurDAG->getTargetConstant(0, DL, MVT::i32);
1512       ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1513       return true;
1514     }
1515   }
1516 
1517   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1518     // (add n0, c1)
1519 
1520     SDValue N0 = Addr.getOperand(0);
1521     SDValue N1 = Addr.getOperand(1);
1522 
1523     // Offsets in vaddr must be positive if range checking is enabled.
1524     //
1525     // The total computation of vaddr + soffset + offset must not overflow.  If
1526     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1527     // overflowing.
1528     //
1529     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1530     // always perform a range check. If a negative vaddr base index was used,
1531     // this would fail the range check. The overall address computation would
1532     // compute a valid address, but this doesn't happen due to the range
1533     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1534     //
1535     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1536     // MUBUF vaddr, but not on older subtargets which can only do this if the
1537     // sign bit is known 0.
1538     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1539     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1540         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1541          CurDAG->SignBitIsZero(N0))) {
1542       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1543       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1544       return true;
1545     }
1546   }
1547 
1548   // (node)
1549   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1550   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1551   return true;
1552 }
1553 
1554 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1555                                                   SDValue Addr,
1556                                                   SDValue &SRsrc,
1557                                                   SDValue &SOffset,
1558                                                   SDValue &Offset) const {
1559   ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
1560   if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1561     return false;
1562 
1563   SDLoc DL(Addr);
1564   MachineFunction &MF = CurDAG->getMachineFunction();
1565   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1566 
1567   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1568 
1569   const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
1570 
1571   // FIXME: Get from MachinePointerInfo? We should only be using the frame
1572   // offset if we know this is in a call sequence.
1573   SOffset = isStackPtrRelative(PtrInfo)
1574                 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
1575                 : CurDAG->getTargetConstant(0, DL, MVT::i32);
1576 
1577   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1578   return true;
1579 }
1580 
1581 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1582                                            SDValue &SOffset, SDValue &Offset,
1583                                            SDValue &GLC, SDValue &SLC,
1584                                            SDValue &TFE, SDValue &DLC,
1585                                            SDValue &SWZ) const {
1586   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1587   const SIInstrInfo *TII =
1588     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1589 
1590   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1591               GLC, SLC, TFE, DLC, SWZ))
1592     return false;
1593 
1594   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1595       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1596       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1597     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1598                     APInt::getAllOnesValue(32).getZExtValue(); // Size
1599     SDLoc DL(Addr);
1600 
1601     const SITargetLowering& Lowering =
1602       *static_cast<const SITargetLowering*>(getTargetLowering());
1603 
1604     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1605     return true;
1606   }
1607   return false;
1608 }
1609 
1610 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1611                                            SDValue &Soffset, SDValue &Offset
1612                                            ) const {
1613   SDValue GLC, SLC, TFE, DLC, SWZ;
1614 
1615   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1616 }
1617 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1618                                            SDValue &Soffset, SDValue &Offset,
1619                                            SDValue &SLC) const {
1620   SDValue GLC, TFE, DLC, SWZ;
1621 
1622   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1623 }
1624 
1625 // Find a load or store from corresponding pattern root.
1626 // Roots may be build_vector, bitconvert or their combinations.
1627 static MemSDNode* findMemSDNode(SDNode *N) {
1628   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1629   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1630     return MN;
1631   assert(isa<BuildVectorSDNode>(N));
1632   for (SDValue V : N->op_values())
1633     if (MemSDNode *MN =
1634           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1635       return MN;
1636   llvm_unreachable("cannot find MemSDNode in the pattern!");
1637 }
1638 
1639 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
1640                                           SDValue &N0, SDValue &N1) {
1641   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
1642       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
1643     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
1644     // (i64 (bitcast (v2i32 (build_vector
1645     //                        (or (extract_vector_elt V, 0), OFFSET),
1646     //                        (extract_vector_elt V, 1)))))
1647     SDValue Lo = Addr.getOperand(0).getOperand(0);
1648     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
1649       SDValue BaseLo = Lo.getOperand(0);
1650       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
1651       // Check that split base (Lo and Hi) are extracted from the same one.
1652       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1653           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1654           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
1655           // Lo is statically extracted from index 0.
1656           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
1657           BaseLo.getConstantOperandVal(1) == 0 &&
1658           // Hi is statically extracted from index 0.
1659           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
1660           BaseHi.getConstantOperandVal(1) == 1) {
1661         N0 = BaseLo.getOperand(0).getOperand(0);
1662         N1 = Lo.getOperand(1);
1663         return true;
1664       }
1665     }
1666   }
1667   return false;
1668 }
1669 
1670 template <bool IsSigned>
1671 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
1672                                           SDValue Addr,
1673                                           SDValue &VAddr,
1674                                           SDValue &Offset) const {
1675   int64_t OffsetVal = 0;
1676 
1677   unsigned AS = findMemSDNode(N)->getAddressSpace();
1678 
1679   if (Subtarget->hasFlatInstOffsets() &&
1680       (!Subtarget->hasFlatSegmentOffsetBug() ||
1681        AS != AMDGPUAS::FLAT_ADDRESS)) {
1682     SDValue N0, N1;
1683     if (CurDAG->isBaseWithConstantOffset(Addr)) {
1684       N0 = Addr.getOperand(0);
1685       N1 = Addr.getOperand(1);
1686     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
1687       assert(N0 && N1 && isa<ConstantSDNode>(N1));
1688     }
1689     if (N0 && N1) {
1690       uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1691 
1692       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1693       if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) {
1694         Addr = N0;
1695         OffsetVal = COffsetVal;
1696       } else {
1697         // If the offset doesn't fit, put the low bits into the offset field and
1698         // add the rest.
1699         //
1700         // For a FLAT instruction the hardware decides whether to access
1701         // global/scratch/shared memory based on the high bits of vaddr,
1702         // ignoring the offset field, so we have to ensure that when we add
1703         // remainder to vaddr it still points into the same underlying object.
1704         // The easiest way to do that is to make sure that we split the offset
1705         // into two pieces that are both >= 0 or both <= 0.
1706 
1707         SDLoc DL(N);
1708         uint64_t RemainderOffset = COffsetVal;
1709         uint64_t ImmField = 0;
1710         const unsigned NumBits = TII->getNumFlatOffsetBits(IsSigned);
1711         if (IsSigned) {
1712           // Use signed division by a power of two to truncate towards 0.
1713           int64_t D = 1LL << (NumBits - 1);
1714           RemainderOffset = (static_cast<int64_t>(COffsetVal) / D) * D;
1715           ImmField = COffsetVal - RemainderOffset;
1716         } else if (static_cast<int64_t>(COffsetVal) >= 0) {
1717           ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits);
1718           RemainderOffset = COffsetVal - ImmField;
1719         }
1720         assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned));
1721         assert(RemainderOffset + ImmField == COffsetVal);
1722 
1723         OffsetVal = ImmField;
1724 
1725         SDValue AddOffsetLo =
1726             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1727         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1728 
1729         if (Addr.getValueType().getSizeInBits() == 32) {
1730           SmallVector<SDValue, 3> Opnds;
1731           Opnds.push_back(N0);
1732           Opnds.push_back(AddOffsetLo);
1733           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1734           if (Subtarget->hasAddNoCarry()) {
1735             AddOp = AMDGPU::V_ADD_U32_e64;
1736             Opnds.push_back(Clamp);
1737           }
1738           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1739         } else {
1740           // TODO: Should this try to use a scalar add pseudo if the base address
1741           // is uniform and saddr is usable?
1742           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1743           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1744 
1745           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1746                                                 DL, MVT::i32, N0, Sub0);
1747           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1748                                                 DL, MVT::i32, N0, Sub1);
1749 
1750           SDValue AddOffsetHi =
1751               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1752 
1753           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1754 
1755           SDNode *Add =
1756               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1757                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1758 
1759           SDNode *Addc = CurDAG->getMachineNode(
1760               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1761               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1762 
1763           SDValue RegSequenceArgs[] = {
1764               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1765               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1766 
1767           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1768                                                 MVT::i64, RegSequenceArgs),
1769                          0);
1770         }
1771       }
1772     }
1773   }
1774 
1775   VAddr = Addr;
1776   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1777   return true;
1778 }
1779 
1780 // If this matches zero_extend i32:x, return x
1781 static SDValue matchZExtFromI32(SDValue Op) {
1782   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1783     return SDValue();
1784 
1785   SDValue ExtSrc = Op.getOperand(0);
1786   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1787 }
1788 
1789 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1790 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1791                                            SDValue Addr,
1792                                            SDValue &SAddr,
1793                                            SDValue &VOffset,
1794                                            SDValue &Offset) const {
1795   int64_t ImmOffset = 0;
1796 
1797   // Match the immediate offset first, which canonically is moved as low as
1798   // possible.
1799   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1800     SDValue LHS = Addr.getOperand(0);
1801     SDValue RHS = Addr.getOperand(1);
1802 
1803     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1804     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1805 
1806     // TODO: Could split larger constant into VGPR offset.
1807     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, true)) {
1808       Addr = LHS;
1809       ImmOffset = COffsetVal;
1810     }
1811   }
1812 
1813   // Match the variable offset.
1814   if (Addr.getOpcode() != ISD::ADD)
1815     return false;
1816 
1817   SDValue LHS = Addr.getOperand(0);
1818   SDValue RHS = Addr.getOperand(1);
1819 
1820   if (!LHS->isDivergent()) {
1821     // add (i64 sgpr), (zero_extend (i32 vgpr))
1822     if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1823       SAddr = LHS;
1824       VOffset = ZextRHS;
1825     }
1826   }
1827 
1828   if (!SAddr && !RHS->isDivergent()) {
1829     // add (zero_extend (i32 vgpr)), (i64 sgpr)
1830     if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1831       SAddr = RHS;
1832       VOffset = ZextLHS;
1833     }
1834   }
1835 
1836   if (!SAddr)
1837     return false;
1838 
1839   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1840   return true;
1841 }
1842 
1843 // Match (32-bit SGPR base) + sext(imm offset)
1844 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *N,
1845                                             SDValue Addr,
1846                                             SDValue &SAddr,
1847                                             SDValue &Offset) const {
1848   if (Addr->isDivergent())
1849     return false;
1850 
1851   SAddr = Addr;
1852   int64_t COffsetVal = 0;
1853 
1854   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1855     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1856     SAddr = Addr.getOperand(0);
1857   }
1858 
1859   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1860     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1861   } else if (SAddr.getOpcode() == ISD::ADD &&
1862              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1863     // Materialize this into a scalar move for scalar address to avoid
1864     // readfirstlane.
1865     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1866     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1867                                               FI->getValueType(0));
1868     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, SDLoc(SAddr),
1869                                            MVT::i32, TFI, SAddr.getOperand(1)),
1870                     0);
1871   }
1872 
1873   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1874 
1875   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1876     int64_t RemainderOffset = COffsetVal;
1877     int64_t ImmField = 0;
1878     const unsigned NumBits = TII->getNumFlatOffsetBits(true);
1879     // Use signed division by a power of two to truncate towards 0.
1880     int64_t D = 1LL << (NumBits - 1);
1881     RemainderOffset = (COffsetVal / D) * D;
1882     ImmField = COffsetVal - RemainderOffset;
1883 
1884     assert(TII->isLegalFLATOffset(ImmField, AMDGPUAS::PRIVATE_ADDRESS, true));
1885     assert(RemainderOffset + ImmField == COffsetVal);
1886 
1887     COffsetVal = ImmField;
1888 
1889     SDLoc DL(N);
1890     SDValue AddOffset =
1891         getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1892     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, DL, MVT::i32,
1893                                            SAddr, AddOffset), 0);
1894   }
1895 
1896   Offset = CurDAG->getTargetConstant(COffsetVal, SDLoc(), MVT::i16);
1897 
1898   return true;
1899 }
1900 
1901 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1902                                           SDValue &Offset, bool &Imm) const {
1903   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1904   if (!C) {
1905     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1906         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1907       Offset = ByteOffsetNode;
1908       Imm = false;
1909       return true;
1910     }
1911     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1912       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1913         Offset = ByteOffsetNode.getOperand(0);
1914         Imm = false;
1915         return true;
1916       }
1917     }
1918     return false;
1919   }
1920 
1921   SDLoc SL(ByteOffsetNode);
1922   // GFX9 and GFX10 have signed byte immediate offsets.
1923   int64_t ByteOffset = C->getSExtValue();
1924   Optional<int64_t> EncodedOffset =
1925       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false);
1926   if (EncodedOffset) {
1927     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1928     Imm = true;
1929     return true;
1930   }
1931 
1932   // SGPR and literal offsets are unsigned.
1933   if (ByteOffset < 0)
1934     return false;
1935 
1936   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1937   if (EncodedOffset) {
1938     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1939     return true;
1940   }
1941 
1942   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1943     return false;
1944 
1945   SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1946   Offset = SDValue(
1947       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1948 
1949   return true;
1950 }
1951 
1952 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1953   if (Addr.getValueType() != MVT::i32)
1954     return Addr;
1955 
1956   // Zero-extend a 32-bit address.
1957   SDLoc SL(Addr);
1958 
1959   const MachineFunction &MF = CurDAG->getMachineFunction();
1960   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1961   unsigned AddrHiVal = Info->get32BitAddressHighBits();
1962   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1963 
1964   const SDValue Ops[] = {
1965     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1966     Addr,
1967     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1968     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1969             0),
1970     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1971   };
1972 
1973   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1974                                         Ops), 0);
1975 }
1976 
1977 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1978                                      SDValue &Offset, bool &Imm) const {
1979   SDLoc SL(Addr);
1980 
1981   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
1982   // wraparound, because s_load instructions perform the addition in 64 bits.
1983   if ((Addr.getValueType() != MVT::i32 ||
1984        Addr->getFlags().hasNoUnsignedWrap())) {
1985     SDValue N0, N1;
1986     // Extract the base and offset if possible.
1987     if (CurDAG->isBaseWithConstantOffset(Addr) ||
1988         Addr.getOpcode() == ISD::ADD) {
1989       N0 = Addr.getOperand(0);
1990       N1 = Addr.getOperand(1);
1991     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
1992       assert(N0 && N1 && isa<ConstantSDNode>(N1));
1993     }
1994     if (N0 && N1) {
1995       if (SelectSMRDOffset(N1, Offset, Imm)) {
1996         SBase = Expand32BitAddress(N0);
1997         return true;
1998       }
1999     }
2000   }
2001   SBase = Expand32BitAddress(Addr);
2002   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
2003   Imm = true;
2004   return true;
2005 }
2006 
2007 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2008                                        SDValue &Offset) const {
2009   bool Imm = false;
2010   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
2011 }
2012 
2013 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2014                                          SDValue &Offset) const {
2015 
2016   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2017 
2018   bool Imm = false;
2019   if (!SelectSMRD(Addr, SBase, Offset, Imm))
2020     return false;
2021 
2022   return !Imm && isa<ConstantSDNode>(Offset);
2023 }
2024 
2025 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2026                                         SDValue &Offset) const {
2027   bool Imm = false;
2028   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
2029          !isa<ConstantSDNode>(Offset);
2030 }
2031 
2032 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
2033                                              SDValue &Offset) const {
2034   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2035     // The immediate offset for S_BUFFER instructions is unsigned.
2036     if (auto Imm =
2037             AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) {
2038       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2039       return true;
2040     }
2041   }
2042 
2043   return false;
2044 }
2045 
2046 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
2047                                                SDValue &Offset) const {
2048   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2049 
2050   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2051     if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget,
2052                                                          C->getZExtValue())) {
2053       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2054       return true;
2055     }
2056   }
2057 
2058   return false;
2059 }
2060 
2061 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2062                                             SDValue &Base,
2063                                             SDValue &Offset) const {
2064   SDLoc DL(Index);
2065 
2066   if (CurDAG->isBaseWithConstantOffset(Index)) {
2067     SDValue N0 = Index.getOperand(0);
2068     SDValue N1 = Index.getOperand(1);
2069     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2070 
2071     // (add n0, c0)
2072     // Don't peel off the offset (c0) if doing so could possibly lead
2073     // the base (n0) to be negative.
2074     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2075     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2076         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2077       Base = N0;
2078       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2079       return true;
2080     }
2081   }
2082 
2083   if (isa<ConstantSDNode>(Index))
2084     return false;
2085 
2086   Base = Index;
2087   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2088   return true;
2089 }
2090 
2091 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
2092                                      SDValue Val, uint32_t Offset,
2093                                      uint32_t Width) {
2094   // Transformation function, pack the offset and width of a BFE into
2095   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2096   // source, bits [5:0] contain the offset and bits [22:16] the width.
2097   uint32_t PackedVal = Offset | (Width << 16);
2098   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2099 
2100   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2101 }
2102 
2103 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2104   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2105   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2106   // Predicate: 0 < b <= c < 32
2107 
2108   const SDValue &Shl = N->getOperand(0);
2109   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2110   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2111 
2112   if (B && C) {
2113     uint32_t BVal = B->getZExtValue();
2114     uint32_t CVal = C->getZExtValue();
2115 
2116     if (0 < BVal && BVal <= CVal && CVal < 32) {
2117       bool Signed = N->getOpcode() == ISD::SRA;
2118       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2119 
2120       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2121                               32 - CVal));
2122       return;
2123     }
2124   }
2125   SelectCode(N);
2126 }
2127 
2128 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2129   switch (N->getOpcode()) {
2130   case ISD::AND:
2131     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2132       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2133       // Predicate: isMask(mask)
2134       const SDValue &Srl = N->getOperand(0);
2135       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2136       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2137 
2138       if (Shift && Mask) {
2139         uint32_t ShiftVal = Shift->getZExtValue();
2140         uint32_t MaskVal = Mask->getZExtValue();
2141 
2142         if (isMask_32(MaskVal)) {
2143           uint32_t WidthVal = countPopulation(MaskVal);
2144 
2145           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2146                                   Srl.getOperand(0), ShiftVal, WidthVal));
2147           return;
2148         }
2149       }
2150     }
2151     break;
2152   case ISD::SRL:
2153     if (N->getOperand(0).getOpcode() == ISD::AND) {
2154       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2155       // Predicate: isMask(mask >> b)
2156       const SDValue &And = N->getOperand(0);
2157       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2158       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2159 
2160       if (Shift && Mask) {
2161         uint32_t ShiftVal = Shift->getZExtValue();
2162         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2163 
2164         if (isMask_32(MaskVal)) {
2165           uint32_t WidthVal = countPopulation(MaskVal);
2166 
2167           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2168                                   And.getOperand(0), ShiftVal, WidthVal));
2169           return;
2170         }
2171       }
2172     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2173       SelectS_BFEFromShifts(N);
2174       return;
2175     }
2176     break;
2177   case ISD::SRA:
2178     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2179       SelectS_BFEFromShifts(N);
2180       return;
2181     }
2182     break;
2183 
2184   case ISD::SIGN_EXTEND_INREG: {
2185     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2186     SDValue Src = N->getOperand(0);
2187     if (Src.getOpcode() != ISD::SRL)
2188       break;
2189 
2190     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2191     if (!Amt)
2192       break;
2193 
2194     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2195     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
2196                             Amt->getZExtValue(), Width));
2197     return;
2198   }
2199   }
2200 
2201   SelectCode(N);
2202 }
2203 
2204 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2205   assert(N->getOpcode() == ISD::BRCOND);
2206   if (!N->hasOneUse())
2207     return false;
2208 
2209   SDValue Cond = N->getOperand(1);
2210   if (Cond.getOpcode() == ISD::CopyToReg)
2211     Cond = Cond.getOperand(2);
2212 
2213   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2214     return false;
2215 
2216   MVT VT = Cond.getOperand(0).getSimpleValueType();
2217   if (VT == MVT::i32)
2218     return true;
2219 
2220   if (VT == MVT::i64) {
2221     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2222 
2223     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2224     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2225   }
2226 
2227   return false;
2228 }
2229 
2230 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2231   SDValue Cond = N->getOperand(1);
2232 
2233   if (Cond.isUndef()) {
2234     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2235                          N->getOperand(2), N->getOperand(0));
2236     return;
2237   }
2238 
2239   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2240   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2241 
2242   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2243   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2244   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2245   SDLoc SL(N);
2246 
2247   if (!UseSCCBr) {
2248     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2249     // analyzed what generates the vcc value, so we do not know whether vcc
2250     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2251     // disabled lanes.
2252     //
2253     // For the case that we select S_CBRANCH_SCC1 and it gets
2254     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2255     // SIInstrInfo::moveToVALU which inserts the S_AND).
2256     //
2257     // We could add an analysis of what generates the vcc value here and omit
2258     // the S_AND when is unnecessary. But it would be better to add a separate
2259     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2260     // catches both cases.
2261     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2262                                                          : AMDGPU::S_AND_B64,
2263                      SL, MVT::i1,
2264                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2265                                                         : AMDGPU::EXEC,
2266                                          MVT::i1),
2267                     Cond),
2268                    0);
2269   }
2270 
2271   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2272   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2273                        N->getOperand(2), // Basic Block
2274                        VCC.getValue(0));
2275 }
2276 
2277 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
2278   MVT VT = N->getSimpleValueType(0);
2279   bool IsFMA = N->getOpcode() == ISD::FMA;
2280   if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
2281                          !Subtarget->hasFmaMixInsts()) ||
2282       ((IsFMA && Subtarget->hasMadMixInsts()) ||
2283        (!IsFMA && Subtarget->hasFmaMixInsts()))) {
2284     SelectCode(N);
2285     return;
2286   }
2287 
2288   SDValue Src0 = N->getOperand(0);
2289   SDValue Src1 = N->getOperand(1);
2290   SDValue Src2 = N->getOperand(2);
2291   unsigned Src0Mods, Src1Mods, Src2Mods;
2292 
2293   // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
2294   // using the conversion from f16.
2295   bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
2296   bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
2297   bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
2298 
2299   assert((IsFMA || !Mode.allFP32Denormals()) &&
2300          "fmad selected with denormals enabled");
2301   // TODO: We can select this with f32 denormals enabled if all the sources are
2302   // converted from f16 (in which case fmad isn't legal).
2303 
2304   if (Sel0 || Sel1 || Sel2) {
2305     // For dummy operands.
2306     SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2307     SDValue Ops[] = {
2308       CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2309       CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2310       CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2311       CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2312       Zero, Zero
2313     };
2314 
2315     CurDAG->SelectNodeTo(N,
2316                          IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2317                          MVT::f32, Ops);
2318   } else {
2319     SelectCode(N);
2320   }
2321 }
2322 
2323 // This is here because there isn't a way to use the generated sub0_sub1 as the
2324 // subreg index to EXTRACT_SUBREG in tablegen.
2325 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2326   MemSDNode *Mem = cast<MemSDNode>(N);
2327   unsigned AS = Mem->getAddressSpace();
2328   if (AS == AMDGPUAS::FLAT_ADDRESS) {
2329     SelectCode(N);
2330     return;
2331   }
2332 
2333   MVT VT = N->getSimpleValueType(0);
2334   bool Is32 = (VT == MVT::i32);
2335   SDLoc SL(N);
2336 
2337   MachineSDNode *CmpSwap = nullptr;
2338   if (Subtarget->hasAddr64()) {
2339     SDValue SRsrc, VAddr, SOffset, Offset, SLC;
2340 
2341     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
2342       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2343         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2344       SDValue CmpVal = Mem->getOperand(2);
2345 
2346       // XXX - Do we care about glue operands?
2347 
2348       SDValue Ops[] = {
2349         CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2350       };
2351 
2352       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2353     }
2354   }
2355 
2356   if (!CmpSwap) {
2357     SDValue SRsrc, SOffset, Offset, SLC;
2358     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
2359       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2360         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2361 
2362       SDValue CmpVal = Mem->getOperand(2);
2363       SDValue Ops[] = {
2364         CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2365       };
2366 
2367       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2368     }
2369   }
2370 
2371   if (!CmpSwap) {
2372     SelectCode(N);
2373     return;
2374   }
2375 
2376   MachineMemOperand *MMO = Mem->getMemOperand();
2377   CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2378 
2379   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2380   SDValue Extract
2381     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2382 
2383   ReplaceUses(SDValue(N, 0), Extract);
2384   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2385   CurDAG->RemoveDeadNode(N);
2386 }
2387 
2388 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2389   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2390   // be copied to an SGPR with readfirstlane.
2391   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2392     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2393 
2394   SDValue Chain = N->getOperand(0);
2395   SDValue Ptr = N->getOperand(2);
2396   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2397   MachineMemOperand *MMO = M->getMemOperand();
2398   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2399 
2400   SDValue Offset;
2401   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2402     SDValue PtrBase = Ptr.getOperand(0);
2403     SDValue PtrOffset = Ptr.getOperand(1);
2404 
2405     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2406     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) {
2407       N = glueCopyToM0(N, PtrBase);
2408       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2409     }
2410   }
2411 
2412   if (!Offset) {
2413     N = glueCopyToM0(N, Ptr);
2414     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2415   }
2416 
2417   SDValue Ops[] = {
2418     Offset,
2419     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2420     Chain,
2421     N->getOperand(N->getNumOperands() - 1) // New glue
2422   };
2423 
2424   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2425   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2426 }
2427 
2428 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2429   switch (IntrID) {
2430   case Intrinsic::amdgcn_ds_gws_init:
2431     return AMDGPU::DS_GWS_INIT;
2432   case Intrinsic::amdgcn_ds_gws_barrier:
2433     return AMDGPU::DS_GWS_BARRIER;
2434   case Intrinsic::amdgcn_ds_gws_sema_v:
2435     return AMDGPU::DS_GWS_SEMA_V;
2436   case Intrinsic::amdgcn_ds_gws_sema_br:
2437     return AMDGPU::DS_GWS_SEMA_BR;
2438   case Intrinsic::amdgcn_ds_gws_sema_p:
2439     return AMDGPU::DS_GWS_SEMA_P;
2440   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2441     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2442   default:
2443     llvm_unreachable("not a gws intrinsic");
2444   }
2445 }
2446 
2447 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2448   if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2449       !Subtarget->hasGWSSemaReleaseAll()) {
2450     // Let this error.
2451     SelectCode(N);
2452     return;
2453   }
2454 
2455   // Chain, intrinsic ID, vsrc, offset
2456   const bool HasVSrc = N->getNumOperands() == 4;
2457   assert(HasVSrc || N->getNumOperands() == 3);
2458 
2459   SDLoc SL(N);
2460   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2461   int ImmOffset = 0;
2462   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2463   MachineMemOperand *MMO = M->getMemOperand();
2464 
2465   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2466   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2467 
2468   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2469   // offset field) % 64. Some versions of the programming guide omit the m0
2470   // part, or claim it's from offset 0.
2471   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2472     // If we have a constant offset, try to use the 0 in m0 as the base.
2473     // TODO: Look into changing the default m0 initialization value. If the
2474     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2475     // the immediate offset.
2476     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2477     ImmOffset = ConstOffset->getZExtValue();
2478   } else {
2479     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2480       ImmOffset = BaseOffset.getConstantOperandVal(1);
2481       BaseOffset = BaseOffset.getOperand(0);
2482     }
2483 
2484     // Prefer to do the shift in an SGPR since it should be possible to use m0
2485     // as the result directly. If it's already an SGPR, it will be eliminated
2486     // later.
2487     SDNode *SGPROffset
2488       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2489                                BaseOffset);
2490     // Shift to offset in m0
2491     SDNode *M0Base
2492       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2493                                SDValue(SGPROffset, 0),
2494                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2495     glueCopyToM0(N, SDValue(M0Base, 0));
2496   }
2497 
2498   SDValue Chain = N->getOperand(0);
2499   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2500 
2501   // TODO: Can this just be removed from the instruction?
2502   SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1);
2503 
2504   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2505   SmallVector<SDValue, 5> Ops;
2506   if (HasVSrc)
2507     Ops.push_back(N->getOperand(2));
2508   Ops.push_back(OffsetField);
2509   Ops.push_back(GDS);
2510   Ops.push_back(Chain);
2511 
2512   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2513   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2514 }
2515 
2516 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2517   if (Subtarget->getLDSBankCount() != 16) {
2518     // This is a single instruction with a pattern.
2519     SelectCode(N);
2520     return;
2521   }
2522 
2523   SDLoc DL(N);
2524 
2525   // This requires 2 instructions. It is possible to write a pattern to support
2526   // this, but the generated isel emitter doesn't correctly deal with multiple
2527   // output instructions using the same physical register input. The copy to m0
2528   // is incorrectly placed before the second instruction.
2529   //
2530   // TODO: Match source modifiers.
2531   //
2532   // def : Pat <
2533   //   (int_amdgcn_interp_p1_f16
2534   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2535   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2536   //                             (i1 timm:$high), M0),
2537   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2538   //       timm:$attrchan, 0,
2539   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2540   //   let Predicates = [has16BankLDS];
2541   // }
2542 
2543   // 16 bank LDS
2544   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2545                                       N->getOperand(5), SDValue());
2546 
2547   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2548 
2549   SDNode *InterpMov =
2550     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2551         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2552         N->getOperand(3),  // Attr
2553         N->getOperand(2),  // Attrchan
2554         ToM0.getValue(1) // In glue
2555   });
2556 
2557   SDNode *InterpP1LV =
2558     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2559         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2560         N->getOperand(1), // Src0
2561         N->getOperand(3), // Attr
2562         N->getOperand(2), // Attrchan
2563         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2564         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2565         N->getOperand(4), // high
2566         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2567         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2568         SDValue(InterpMov, 1)
2569   });
2570 
2571   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2572 }
2573 
2574 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2575   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2576   switch (IntrID) {
2577   case Intrinsic::amdgcn_ds_append:
2578   case Intrinsic::amdgcn_ds_consume: {
2579     if (N->getValueType(0) != MVT::i32)
2580       break;
2581     SelectDSAppendConsume(N, IntrID);
2582     return;
2583   }
2584   }
2585 
2586   SelectCode(N);
2587 }
2588 
2589 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2590   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2591   unsigned Opcode;
2592   switch (IntrID) {
2593   case Intrinsic::amdgcn_wqm:
2594     Opcode = AMDGPU::WQM;
2595     break;
2596   case Intrinsic::amdgcn_softwqm:
2597     Opcode = AMDGPU::SOFT_WQM;
2598     break;
2599   case Intrinsic::amdgcn_wwm:
2600     Opcode = AMDGPU::WWM;
2601     break;
2602   case Intrinsic::amdgcn_interp_p1_f16:
2603     SelectInterpP1F16(N);
2604     return;
2605   default:
2606     SelectCode(N);
2607     return;
2608   }
2609 
2610   SDValue Src = N->getOperand(1);
2611   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2612 }
2613 
2614 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2615   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2616   switch (IntrID) {
2617   case Intrinsic::amdgcn_ds_gws_init:
2618   case Intrinsic::amdgcn_ds_gws_barrier:
2619   case Intrinsic::amdgcn_ds_gws_sema_v:
2620   case Intrinsic::amdgcn_ds_gws_sema_br:
2621   case Intrinsic::amdgcn_ds_gws_sema_p:
2622   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2623     SelectDS_GWS(N, IntrID);
2624     return;
2625   default:
2626     break;
2627   }
2628 
2629   SelectCode(N);
2630 }
2631 
2632 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2633                                             unsigned &Mods) const {
2634   Mods = 0;
2635   Src = In;
2636 
2637   if (Src.getOpcode() == ISD::FNEG) {
2638     Mods |= SISrcMods::NEG;
2639     Src = Src.getOperand(0);
2640   }
2641 
2642   if (Src.getOpcode() == ISD::FABS) {
2643     Mods |= SISrcMods::ABS;
2644     Src = Src.getOperand(0);
2645   }
2646 
2647   return true;
2648 }
2649 
2650 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2651                                         SDValue &SrcMods) const {
2652   unsigned Mods;
2653   if (SelectVOP3ModsImpl(In, Src, Mods)) {
2654     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2655     return true;
2656   }
2657 
2658   return false;
2659 }
2660 
2661 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2662                                              SDValue &SrcMods) const {
2663   SelectVOP3Mods(In, Src, SrcMods);
2664   return isNoNanSrc(Src);
2665 }
2666 
2667 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2668   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2669     return false;
2670 
2671   Src = In;
2672   return true;
2673 }
2674 
2675 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2676                                          SDValue &SrcMods, SDValue &Clamp,
2677                                          SDValue &Omod) const {
2678   SDLoc DL(In);
2679   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2680   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2681 
2682   return SelectVOP3Mods(In, Src, SrcMods);
2683 }
2684 
2685 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2686                                          SDValue &Clamp, SDValue &Omod) const {
2687   Src = In;
2688 
2689   SDLoc DL(In);
2690   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2691   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2692 
2693   return true;
2694 }
2695 
2696 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2697                                          SDValue &SrcMods) const {
2698   unsigned Mods = 0;
2699   Src = In;
2700 
2701   if (Src.getOpcode() == ISD::FNEG) {
2702     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2703     Src = Src.getOperand(0);
2704   }
2705 
2706   if (Src.getOpcode() == ISD::BUILD_VECTOR) {
2707     unsigned VecMods = Mods;
2708 
2709     SDValue Lo = stripBitcast(Src.getOperand(0));
2710     SDValue Hi = stripBitcast(Src.getOperand(1));
2711 
2712     if (Lo.getOpcode() == ISD::FNEG) {
2713       Lo = stripBitcast(Lo.getOperand(0));
2714       Mods ^= SISrcMods::NEG;
2715     }
2716 
2717     if (Hi.getOpcode() == ISD::FNEG) {
2718       Hi = stripBitcast(Hi.getOperand(0));
2719       Mods ^= SISrcMods::NEG_HI;
2720     }
2721 
2722     if (isExtractHiElt(Lo, Lo))
2723       Mods |= SISrcMods::OP_SEL_0;
2724 
2725     if (isExtractHiElt(Hi, Hi))
2726       Mods |= SISrcMods::OP_SEL_1;
2727 
2728     Lo = stripExtractLoElt(Lo);
2729     Hi = stripExtractLoElt(Hi);
2730 
2731     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2732       // Really a scalar input. Just select from the low half of the register to
2733       // avoid packing.
2734 
2735       Src = Lo;
2736       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2737       return true;
2738     }
2739 
2740     Mods = VecMods;
2741   }
2742 
2743   // Packed instructions do not have abs modifiers.
2744   Mods |= SISrcMods::OP_SEL_1;
2745 
2746   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2747   return true;
2748 }
2749 
2750 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2751                                          SDValue &SrcMods) const {
2752   Src = In;
2753   // FIXME: Handle op_sel
2754   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2755   return true;
2756 }
2757 
2758 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2759                                              SDValue &SrcMods) const {
2760   // FIXME: Handle op_sel
2761   return SelectVOP3Mods(In, Src, SrcMods);
2762 }
2763 
2764 // The return value is not whether the match is possible (which it always is),
2765 // but whether or not it a conversion is really used.
2766 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2767                                                    unsigned &Mods) const {
2768   Mods = 0;
2769   SelectVOP3ModsImpl(In, Src, Mods);
2770 
2771   if (Src.getOpcode() == ISD::FP_EXTEND) {
2772     Src = Src.getOperand(0);
2773     assert(Src.getValueType() == MVT::f16);
2774     Src = stripBitcast(Src);
2775 
2776     // Be careful about folding modifiers if we already have an abs. fneg is
2777     // applied last, so we don't want to apply an earlier fneg.
2778     if ((Mods & SISrcMods::ABS) == 0) {
2779       unsigned ModsTmp;
2780       SelectVOP3ModsImpl(Src, Src, ModsTmp);
2781 
2782       if ((ModsTmp & SISrcMods::NEG) != 0)
2783         Mods ^= SISrcMods::NEG;
2784 
2785       if ((ModsTmp & SISrcMods::ABS) != 0)
2786         Mods |= SISrcMods::ABS;
2787     }
2788 
2789     // op_sel/op_sel_hi decide the source type and source.
2790     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2791     // If the sources's op_sel is set, it picks the high half of the source
2792     // register.
2793 
2794     Mods |= SISrcMods::OP_SEL_1;
2795     if (isExtractHiElt(Src, Src)) {
2796       Mods |= SISrcMods::OP_SEL_0;
2797 
2798       // TODO: Should we try to look for neg/abs here?
2799     }
2800 
2801     return true;
2802   }
2803 
2804   return false;
2805 }
2806 
2807 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2808                                                SDValue &SrcMods) const {
2809   unsigned Mods = 0;
2810   SelectVOP3PMadMixModsImpl(In, Src, Mods);
2811   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2812   return true;
2813 }
2814 
2815 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2816   if (In.isUndef())
2817     return CurDAG->getUNDEF(MVT::i32);
2818 
2819   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2820     SDLoc SL(In);
2821     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2822   }
2823 
2824   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2825     SDLoc SL(In);
2826     return CurDAG->getConstant(
2827       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2828   }
2829 
2830   SDValue Src;
2831   if (isExtractHiElt(In, Src))
2832     return Src;
2833 
2834   return SDValue();
2835 }
2836 
2837 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2838   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2839 
2840   const SIRegisterInfo *SIRI =
2841     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2842   const SIInstrInfo * SII =
2843     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2844 
2845   unsigned Limit = 0;
2846   bool AllUsesAcceptSReg = true;
2847   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2848     Limit < 10 && U != E; ++U, ++Limit) {
2849     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2850 
2851     // If the register class is unknown, it could be an unknown
2852     // register class that needs to be an SGPR, e.g. an inline asm
2853     // constraint
2854     if (!RC || SIRI->isSGPRClass(RC))
2855       return false;
2856 
2857     if (RC != &AMDGPU::VS_32RegClass) {
2858       AllUsesAcceptSReg = false;
2859       SDNode * User = *U;
2860       if (User->isMachineOpcode()) {
2861         unsigned Opc = User->getMachineOpcode();
2862         MCInstrDesc Desc = SII->get(Opc);
2863         if (Desc.isCommutable()) {
2864           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2865           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2866           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2867             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2868             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2869             if (CommutedRC == &AMDGPU::VS_32RegClass)
2870               AllUsesAcceptSReg = true;
2871           }
2872         }
2873       }
2874       // If "AllUsesAcceptSReg == false" so far we haven't suceeded
2875       // commuting current user. This means have at least one use
2876       // that strictly require VGPR. Thus, we will not attempt to commute
2877       // other user instructions.
2878       if (!AllUsesAcceptSReg)
2879         break;
2880     }
2881   }
2882   return !AllUsesAcceptSReg && (Limit < 10);
2883 }
2884 
2885 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2886   auto Ld = cast<LoadSDNode>(N);
2887 
2888   return Ld->getAlignment() >= 4 &&
2889         (
2890           (
2891             (
2892               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS       ||
2893               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT
2894             )
2895             &&
2896             !N->isDivergent()
2897           )
2898           ||
2899           (
2900             Subtarget->getScalarizeGlobalBehavior() &&
2901             Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2902             Ld->isSimple() &&
2903             !N->isDivergent() &&
2904             static_cast<const SITargetLowering *>(
2905               getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)
2906           )
2907         );
2908 }
2909 
2910 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2911   const AMDGPUTargetLowering& Lowering =
2912     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2913   bool IsModified = false;
2914   do {
2915     IsModified = false;
2916 
2917     // Go over all selected nodes and try to fold them a bit more
2918     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2919     while (Position != CurDAG->allnodes_end()) {
2920       SDNode *Node = &*Position++;
2921       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
2922       if (!MachineNode)
2923         continue;
2924 
2925       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
2926       if (ResNode != Node) {
2927         if (ResNode)
2928           ReplaceUses(Node, ResNode);
2929         IsModified = true;
2930       }
2931     }
2932     CurDAG->RemoveDeadNodes();
2933   } while (IsModified);
2934 }
2935 
2936 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
2937   Subtarget = &MF.getSubtarget<R600Subtarget>();
2938   return SelectionDAGISel::runOnMachineFunction(MF);
2939 }
2940 
2941 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
2942   if (!N->readMem())
2943     return false;
2944   if (CbId == -1)
2945     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2946            N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
2947 
2948   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
2949 }
2950 
2951 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
2952                                                          SDValue& IntPtr) {
2953   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
2954     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
2955                                        true);
2956     return true;
2957   }
2958   return false;
2959 }
2960 
2961 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
2962     SDValue& BaseReg, SDValue &Offset) {
2963   if (!isa<ConstantSDNode>(Addr)) {
2964     BaseReg = Addr;
2965     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
2966     return true;
2967   }
2968   return false;
2969 }
2970 
2971 void R600DAGToDAGISel::Select(SDNode *N) {
2972   unsigned int Opc = N->getOpcode();
2973   if (N->isMachineOpcode()) {
2974     N->setNodeId(-1);
2975     return;   // Already selected.
2976   }
2977 
2978   switch (Opc) {
2979   default: break;
2980   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
2981   case ISD::SCALAR_TO_VECTOR:
2982   case ISD::BUILD_VECTOR: {
2983     EVT VT = N->getValueType(0);
2984     unsigned NumVectorElts = VT.getVectorNumElements();
2985     unsigned RegClassID;
2986     // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
2987     // that adds a 128 bits reg copy when going through TwoAddressInstructions
2988     // pass. We want to avoid 128 bits copies as much as possible because they
2989     // can't be bundled by our scheduler.
2990     switch(NumVectorElts) {
2991     case 2: RegClassID = R600::R600_Reg64RegClassID; break;
2992     case 4:
2993       if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
2994         RegClassID = R600::R600_Reg128VerticalRegClassID;
2995       else
2996         RegClassID = R600::R600_Reg128RegClassID;
2997       break;
2998     default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
2999     }
3000     SelectBuildVector(N, RegClassID);
3001     return;
3002   }
3003   }
3004 
3005   SelectCode(N);
3006 }
3007 
3008 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
3009                                           SDValue &Offset) {
3010   ConstantSDNode *C;
3011   SDLoc DL(Addr);
3012 
3013   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
3014     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3015     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3016   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
3017              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
3018     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3019     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3020   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
3021             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
3022     Base = Addr.getOperand(0);
3023     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3024   } else {
3025     Base = Addr;
3026     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
3027   }
3028 
3029   return true;
3030 }
3031 
3032 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
3033                                           SDValue &Offset) {
3034   ConstantSDNode *IMMOffset;
3035 
3036   if (Addr.getOpcode() == ISD::ADD
3037       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
3038       && isInt<16>(IMMOffset->getZExtValue())) {
3039 
3040       Base = Addr.getOperand(0);
3041       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3042                                          MVT::i32);
3043       return true;
3044   // If the pointer address is constant, we can move it to the offset field.
3045   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
3046              && isInt<16>(IMMOffset->getZExtValue())) {
3047     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
3048                                   SDLoc(CurDAG->getEntryNode()),
3049                                   R600::ZERO, MVT::i32);
3050     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3051                                        MVT::i32);
3052     return true;
3053   }
3054 
3055   // Default case, no offset
3056   Base = Addr;
3057   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
3058   return true;
3059 }
3060