1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPUTargetMachine.h"
16 #include "SIMachineFunctionInfo.h"
17 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
18 #include "llvm/Analysis/ValueTracking.h"
19 #include "llvm/CodeGen/FunctionLoweringInfo.h"
20 #include "llvm/CodeGen/SelectionDAG.h"
21 #include "llvm/CodeGen/SelectionDAGISel.h"
22 #include "llvm/CodeGen/SelectionDAGNodes.h"
23 #include "llvm/IR/IntrinsicsAMDGPU.h"
24 #include "llvm/InitializePasses.h"
25 
26 #ifdef EXPENSIVE_CHECKS
27 #include "llvm/Analysis/LoopInfo.h"
28 #include "llvm/IR/Dominators.h"
29 #endif
30 
31 #define DEBUG_TYPE "isel"
32 
33 using namespace llvm;
34 
35 namespace llvm {
36 
37 class R600InstrInfo;
38 
39 } // end namespace llvm
40 
41 //===----------------------------------------------------------------------===//
42 // Instruction Selector Implementation
43 //===----------------------------------------------------------------------===//
44 
45 namespace {
46 
47 static bool isNullConstantOrUndef(SDValue V) {
48   if (V.isUndef())
49     return true;
50 
51   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
52   return Const != nullptr && Const->isNullValue();
53 }
54 
55 static bool getConstantValue(SDValue N, uint32_t &Out) {
56   // This is only used for packed vectors, where ussing 0 for undef should
57   // always be good.
58   if (N.isUndef()) {
59     Out = 0;
60     return true;
61   }
62 
63   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
64     Out = C->getAPIntValue().getSExtValue();
65     return true;
66   }
67 
68   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
69     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
70     return true;
71   }
72 
73   return false;
74 }
75 
76 // TODO: Handle undef as zero
77 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
78                                  bool Negate = false) {
79   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
80   uint32_t LHSVal, RHSVal;
81   if (getConstantValue(N->getOperand(0), LHSVal) &&
82       getConstantValue(N->getOperand(1), RHSVal)) {
83     SDLoc SL(N);
84     uint32_t K = Negate ?
85       (-LHSVal & 0xffff) | (-RHSVal << 16) :
86       (LHSVal & 0xffff) | (RHSVal << 16);
87     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
88                               DAG.getTargetConstant(K, SL, MVT::i32));
89   }
90 
91   return nullptr;
92 }
93 
94 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
95   return packConstantV2I16(N, DAG, true);
96 }
97 
98 /// AMDGPU specific code to select AMDGPU machine instructions for
99 /// SelectionDAG operations.
100 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
101   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
102   // make the right decision when generating code for different targets.
103   const GCNSubtarget *Subtarget;
104 
105   // Default FP mode for the current function.
106   AMDGPU::SIModeRegisterDefaults Mode;
107 
108   bool EnableLateStructurizeCFG;
109 
110 public:
111   explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
112                               CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
113     : SelectionDAGISel(*TM, OptLevel) {
114     EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
115   }
116   ~AMDGPUDAGToDAGISel() override = default;
117 
118   void getAnalysisUsage(AnalysisUsage &AU) const override {
119     AU.addRequired<AMDGPUArgumentUsageInfo>();
120     AU.addRequired<LegacyDivergenceAnalysis>();
121 #ifdef EXPENSIVE_CHECKS
122     AU.addRequired<DominatorTreeWrapperPass>();
123     AU.addRequired<LoopInfoWrapperPass>();
124 #endif
125     SelectionDAGISel::getAnalysisUsage(AU);
126   }
127 
128   bool matchLoadD16FromBuildVector(SDNode *N) const;
129 
130   bool runOnMachineFunction(MachineFunction &MF) override;
131   void PreprocessISelDAG() override;
132   void Select(SDNode *N) override;
133   StringRef getPassName() const override;
134   void PostprocessISelDAG() override;
135 
136 protected:
137   void SelectBuildVector(SDNode *N, unsigned RegClassID);
138 
139 private:
140   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
141   bool isNoNanSrc(SDValue N) const;
142   bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
143   bool isNegInlineImmediate(const SDNode *N) const {
144     return isInlineImmediate(N, true);
145   }
146 
147   bool isInlineImmediate16(int64_t Imm) const {
148     return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());
149   }
150 
151   bool isInlineImmediate32(int64_t Imm) const {
152     return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());
153   }
154 
155   bool isInlineImmediate64(int64_t Imm) const {
156     return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());
157   }
158 
159   bool isInlineImmediate(const APFloat &Imm) const {
160     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
161   }
162 
163   bool isVGPRImm(const SDNode *N) const;
164   bool isUniformLoad(const SDNode *N) const;
165   bool isUniformBr(const SDNode *N) const;
166 
167   bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
168                                   SDValue &RHS) const;
169 
170   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
171 
172   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
173   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
174   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
175 
176   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
177   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
178   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
179   bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
180   bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
181                         unsigned Size) const;
182   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
183   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
184                                  SDValue &Offset1) const;
185   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
186                                   SDValue &Offset1) const;
187   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
188                           SDValue &Offset1, unsigned Size) const;
189   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
190                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
191                    SDValue &Idxen, SDValue &Addr64) const;
192   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
193                          SDValue &SOffset, SDValue &Offset) const;
194   bool SelectMUBUFScratchOffen(SDNode *Parent,
195                                SDValue Addr, SDValue &RSrc, SDValue &VAddr,
196                                SDValue &SOffset, SDValue &ImmOffset) const;
197   bool SelectMUBUFScratchOffset(SDNode *Parent,
198                                 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
199                                 SDValue &Offset) const;
200 
201   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
202                          SDValue &Offset) const;
203 
204   bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
205                             SDValue &Offset, uint64_t FlatVariant) const;
206   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
207                         SDValue &Offset) const;
208   bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
209                           SDValue &Offset) const;
210   bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
211                            SDValue &Offset) const;
212   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
213                          SDValue &VOffset, SDValue &Offset) const;
214   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
215                           SDValue &Offset) const;
216 
217   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
218                         bool &Imm) const;
219   SDValue Expand32BitAddress(SDValue Addr) const;
220   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
221                   bool &Imm) const;
222   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
223   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
224   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
225   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
226   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
227   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
228 
229   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
230   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
231                           bool AllowAbs = true) const;
232   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
233   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
235   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
236                        SDValue &Clamp, SDValue &Omod) const;
237   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
238                         SDValue &Clamp, SDValue &Omod) const;
239   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
240                          SDValue &Clamp, SDValue &Omod) const;
241 
242   bool SelectVOP3OMods(SDValue In, SDValue &Src,
243                        SDValue &Clamp, SDValue &Omod) const;
244 
245   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
246 
247   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
248 
249   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
250   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
251   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
252 
253   SDValue getHi16Elt(SDValue In) const;
254 
255   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
256 
257   void SelectADD_SUB_I64(SDNode *N);
258   void SelectAddcSubb(SDNode *N);
259   void SelectUADDO_USUBO(SDNode *N);
260   void SelectDIV_SCALE(SDNode *N);
261   void SelectMAD_64_32(SDNode *N);
262   void SelectFMA_W_CHAIN(SDNode *N);
263   void SelectFMUL_W_CHAIN(SDNode *N);
264 
265   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
266                    uint32_t Offset, uint32_t Width);
267   void SelectS_BFEFromShifts(SDNode *N);
268   void SelectS_BFE(SDNode *N);
269   bool isCBranchSCC(const SDNode *N) const;
270   void SelectBRCOND(SDNode *N);
271   void SelectFMAD_FMA(SDNode *N);
272   void SelectATOMIC_CMP_SWAP(SDNode *N);
273   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
274   void SelectDS_GWS(SDNode *N, unsigned IntrID);
275   void SelectInterpP1F16(SDNode *N);
276   void SelectINTRINSIC_W_CHAIN(SDNode *N);
277   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
278   void SelectINTRINSIC_VOID(SDNode *N);
279 
280 protected:
281   // Include the pieces autogenerated from the target description.
282 #include "AMDGPUGenDAGISel.inc"
283 };
284 
285 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
286   const R600Subtarget *Subtarget;
287 
288   bool isConstantLoad(const MemSDNode *N, int cbID) const;
289   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
290   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
291                                        SDValue& Offset);
292 public:
293   explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
294       AMDGPUDAGToDAGISel(TM, OptLevel) {}
295 
296   void Select(SDNode *N) override;
297 
298   bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
299                           SDValue &Offset) override;
300   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
301                           SDValue &Offset) override;
302 
303   bool runOnMachineFunction(MachineFunction &MF) override;
304 
305   void PreprocessISelDAG() override {}
306 
307 protected:
308   // Include the pieces autogenerated from the target description.
309 #include "R600GenDAGISel.inc"
310 };
311 
312 static SDValue stripBitcast(SDValue Val) {
313   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
314 }
315 
316 // Figure out if this is really an extract of the high 16-bits of a dword.
317 static bool isExtractHiElt(SDValue In, SDValue &Out) {
318   In = stripBitcast(In);
319 
320   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
321     if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
322       if (!Idx->isOne())
323         return false;
324       Out = In.getOperand(0);
325       return true;
326     }
327   }
328 
329   if (In.getOpcode() != ISD::TRUNCATE)
330     return false;
331 
332   SDValue Srl = In.getOperand(0);
333   if (Srl.getOpcode() == ISD::SRL) {
334     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
335       if (ShiftAmt->getZExtValue() == 16) {
336         Out = stripBitcast(Srl.getOperand(0));
337         return true;
338       }
339     }
340   }
341 
342   return false;
343 }
344 
345 // Look through operations that obscure just looking at the low 16-bits of the
346 // same register.
347 static SDValue stripExtractLoElt(SDValue In) {
348   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
349     if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
350       if (Idx->isNullValue() && In.getValueSizeInBits() <= 32)
351         return In.getOperand(0);
352     }
353   }
354 
355   if (In.getOpcode() == ISD::TRUNCATE) {
356     SDValue Src = In.getOperand(0);
357     if (Src.getValueType().getSizeInBits() == 32)
358       return stripBitcast(Src);
359   }
360 
361   return In;
362 }
363 
364 }  // end anonymous namespace
365 
366 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
367                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
368 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
369 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
370 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
371 #ifdef EXPENSIVE_CHECKS
372 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
373 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
374 #endif
375 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
376                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
377 
378 /// This pass converts a legalized DAG into a AMDGPU-specific
379 // DAG, ready for instruction scheduling.
380 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
381                                         CodeGenOpt::Level OptLevel) {
382   return new AMDGPUDAGToDAGISel(TM, OptLevel);
383 }
384 
385 /// This pass converts a legalized DAG into a R600-specific
386 // DAG, ready for instruction scheduling.
387 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
388                                       CodeGenOpt::Level OptLevel) {
389   return new R600DAGToDAGISel(TM, OptLevel);
390 }
391 
392 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
393 #ifdef EXPENSIVE_CHECKS
394   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
395   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
396   for (auto &L : LI->getLoopsInPreorder()) {
397     assert(L->isLCSSAForm(DT));
398   }
399 #endif
400   Subtarget = &MF.getSubtarget<GCNSubtarget>();
401   Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
402   return SelectionDAGISel::runOnMachineFunction(MF);
403 }
404 
405 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
406   assert(Subtarget->d16PreservesUnusedBits());
407   MVT VT = N->getValueType(0).getSimpleVT();
408   if (VT != MVT::v2i16 && VT != MVT::v2f16)
409     return false;
410 
411   SDValue Lo = N->getOperand(0);
412   SDValue Hi = N->getOperand(1);
413 
414   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
415 
416   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
417   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
418   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
419 
420   // Need to check for possible indirect dependencies on the other half of the
421   // vector to avoid introducing a cycle.
422   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
423     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
424 
425     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
426     SDValue Ops[] = {
427       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
428     };
429 
430     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
431     if (LdHi->getMemoryVT() == MVT::i8) {
432       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
433         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
434     } else {
435       assert(LdHi->getMemoryVT() == MVT::i16);
436     }
437 
438     SDValue NewLoadHi =
439       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
440                                   Ops, LdHi->getMemoryVT(),
441                                   LdHi->getMemOperand());
442 
443     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
444     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
445     return true;
446   }
447 
448   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
449   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
450   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
451   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
452   if (LdLo && Lo.hasOneUse()) {
453     SDValue TiedIn = getHi16Elt(Hi);
454     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
455       return false;
456 
457     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
458     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
459     if (LdLo->getMemoryVT() == MVT::i8) {
460       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
461         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
462     } else {
463       assert(LdLo->getMemoryVT() == MVT::i16);
464     }
465 
466     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
467 
468     SDValue Ops[] = {
469       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
470     };
471 
472     SDValue NewLoadLo =
473       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
474                                   Ops, LdLo->getMemoryVT(),
475                                   LdLo->getMemOperand());
476 
477     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
478     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
479     return true;
480   }
481 
482   return false;
483 }
484 
485 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
486   if (!Subtarget->d16PreservesUnusedBits())
487     return;
488 
489   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
490 
491   bool MadeChange = false;
492   while (Position != CurDAG->allnodes_begin()) {
493     SDNode *N = &*--Position;
494     if (N->use_empty())
495       continue;
496 
497     switch (N->getOpcode()) {
498     case ISD::BUILD_VECTOR:
499       MadeChange |= matchLoadD16FromBuildVector(N);
500       break;
501     default:
502       break;
503     }
504   }
505 
506   if (MadeChange) {
507     CurDAG->RemoveDeadNodes();
508     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
509                CurDAG->dump(););
510   }
511 }
512 
513 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
514   if (TM.Options.NoNaNsFPMath)
515     return true;
516 
517   // TODO: Move into isKnownNeverNaN
518   if (N->getFlags().hasNoNaNs())
519     return true;
520 
521   return CurDAG->isKnownNeverNaN(N);
522 }
523 
524 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
525                                            bool Negated) const {
526   if (N->isUndef())
527     return true;
528 
529   const SIInstrInfo *TII = Subtarget->getInstrInfo();
530   if (Negated) {
531     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
532       return TII->isInlineConstant(-C->getAPIntValue());
533 
534     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
535       return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
536 
537   } else {
538     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
539       return TII->isInlineConstant(C->getAPIntValue());
540 
541     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
542       return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
543   }
544 
545   return false;
546 }
547 
548 /// Determine the register class for \p OpNo
549 /// \returns The register class of the virtual register that will be used for
550 /// the given operand number \OpNo or NULL if the register class cannot be
551 /// determined.
552 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
553                                                           unsigned OpNo) const {
554   if (!N->isMachineOpcode()) {
555     if (N->getOpcode() == ISD::CopyToReg) {
556       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
557       if (Reg.isVirtual()) {
558         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
559         return MRI.getRegClass(Reg);
560       }
561 
562       const SIRegisterInfo *TRI
563         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
564       return TRI->getPhysRegClass(Reg);
565     }
566 
567     return nullptr;
568   }
569 
570   switch (N->getMachineOpcode()) {
571   default: {
572     const MCInstrDesc &Desc =
573         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
574     unsigned OpIdx = Desc.getNumDefs() + OpNo;
575     if (OpIdx >= Desc.getNumOperands())
576       return nullptr;
577     int RegClass = Desc.OpInfo[OpIdx].RegClass;
578     if (RegClass == -1)
579       return nullptr;
580 
581     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
582   }
583   case AMDGPU::REG_SEQUENCE: {
584     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
585     const TargetRegisterClass *SuperRC =
586         Subtarget->getRegisterInfo()->getRegClass(RCID);
587 
588     SDValue SubRegOp = N->getOperand(OpNo + 1);
589     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
590     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
591                                                               SubRegIdx);
592   }
593   }
594 }
595 
596 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
597                                          SDValue Glue) const {
598   SmallVector <SDValue, 8> Ops;
599   Ops.push_back(NewChain); // Replace the chain.
600   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
601     Ops.push_back(N->getOperand(i));
602 
603   Ops.push_back(Glue);
604   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
605 }
606 
607 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
608   const SITargetLowering& Lowering =
609     *static_cast<const SITargetLowering*>(getTargetLowering());
610 
611   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
612 
613   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
614   return glueCopyToOp(N, M0, M0.getValue(1));
615 }
616 
617 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
618   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
619   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
620     if (Subtarget->ldsRequiresM0Init())
621       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
622   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
623     MachineFunction &MF = CurDAG->getMachineFunction();
624     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
625     return
626         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
627   }
628   return N;
629 }
630 
631 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
632                                                   EVT VT) const {
633   SDNode *Lo = CurDAG->getMachineNode(
634       AMDGPU::S_MOV_B32, DL, MVT::i32,
635       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
636   SDNode *Hi =
637       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
638                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
639   const SDValue Ops[] = {
640       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
641       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
642       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
643 
644   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
645 }
646 
647 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
648   EVT VT = N->getValueType(0);
649   unsigned NumVectorElts = VT.getVectorNumElements();
650   EVT EltVT = VT.getVectorElementType();
651   SDLoc DL(N);
652   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
653 
654   if (NumVectorElts == 1) {
655     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
656                          RegClass);
657     return;
658   }
659 
660   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
661                                   "supported yet");
662   // 32 = Max Num Vector Elements
663   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
664   // 1 = Vector Register Class
665   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
666 
667   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
668                Triple::amdgcn;
669   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
670   bool IsRegSeq = true;
671   unsigned NOps = N->getNumOperands();
672   for (unsigned i = 0; i < NOps; i++) {
673     // XXX: Why is this here?
674     if (isa<RegisterSDNode>(N->getOperand(i))) {
675       IsRegSeq = false;
676       break;
677     }
678     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
679                          : R600RegisterInfo::getSubRegFromChannel(i);
680     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
681     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
682   }
683   if (NOps != NumVectorElts) {
684     // Fill in the missing undef elements if this was a scalar_to_vector.
685     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
686     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
687                                                    DL, EltVT);
688     for (unsigned i = NOps; i < NumVectorElts; ++i) {
689       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
690                            : R600RegisterInfo::getSubRegFromChannel(i);
691       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
692       RegSeqArgs[1 + (2 * i) + 1] =
693           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
694     }
695   }
696 
697   if (!IsRegSeq)
698     SelectCode(N);
699   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
700 }
701 
702 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
703   unsigned int Opc = N->getOpcode();
704   if (N->isMachineOpcode()) {
705     N->setNodeId(-1);
706     return;   // Already selected.
707   }
708 
709   // isa<MemSDNode> almost works but is slightly too permissive for some DS
710   // intrinsics.
711   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
712       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
713        Opc == ISD::ATOMIC_LOAD_FADD ||
714        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
715        Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
716     N = glueCopyToM0LDSInit(N);
717     SelectCode(N);
718     return;
719   }
720 
721   switch (Opc) {
722   default:
723     break;
724   // We are selecting i64 ADD here instead of custom lower it during
725   // DAG legalization, so we can fold some i64 ADDs used for address
726   // calculation into the LOAD and STORE instructions.
727   case ISD::ADDC:
728   case ISD::ADDE:
729   case ISD::SUBC:
730   case ISD::SUBE: {
731     if (N->getValueType(0) != MVT::i64)
732       break;
733 
734     SelectADD_SUB_I64(N);
735     return;
736   }
737   case ISD::ADDCARRY:
738   case ISD::SUBCARRY:
739     if (N->getValueType(0) != MVT::i32)
740       break;
741 
742     SelectAddcSubb(N);
743     return;
744   case ISD::UADDO:
745   case ISD::USUBO: {
746     SelectUADDO_USUBO(N);
747     return;
748   }
749   case AMDGPUISD::FMUL_W_CHAIN: {
750     SelectFMUL_W_CHAIN(N);
751     return;
752   }
753   case AMDGPUISD::FMA_W_CHAIN: {
754     SelectFMA_W_CHAIN(N);
755     return;
756   }
757 
758   case ISD::SCALAR_TO_VECTOR:
759   case ISD::BUILD_VECTOR: {
760     EVT VT = N->getValueType(0);
761     unsigned NumVectorElts = VT.getVectorNumElements();
762     if (VT.getScalarSizeInBits() == 16) {
763       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
764         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
765           ReplaceNode(N, Packed);
766           return;
767         }
768       }
769 
770       break;
771     }
772 
773     assert(VT.getVectorElementType().bitsEq(MVT::i32));
774     unsigned RegClassID =
775         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
776     SelectBuildVector(N, RegClassID);
777     return;
778   }
779   case ISD::BUILD_PAIR: {
780     SDValue RC, SubReg0, SubReg1;
781     SDLoc DL(N);
782     if (N->getValueType(0) == MVT::i128) {
783       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
784       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
785       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
786     } else if (N->getValueType(0) == MVT::i64) {
787       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
788       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
789       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
790     } else {
791       llvm_unreachable("Unhandled value type for BUILD_PAIR");
792     }
793     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
794                             N->getOperand(1), SubReg1 };
795     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
796                                           N->getValueType(0), Ops));
797     return;
798   }
799 
800   case ISD::Constant:
801   case ISD::ConstantFP: {
802     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
803       break;
804 
805     uint64_t Imm;
806     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
807       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
808     else {
809       ConstantSDNode *C = cast<ConstantSDNode>(N);
810       Imm = C->getZExtValue();
811     }
812 
813     SDLoc DL(N);
814     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
815     return;
816   }
817   case AMDGPUISD::BFE_I32:
818   case AMDGPUISD::BFE_U32: {
819     // There is a scalar version available, but unlike the vector version which
820     // has a separate operand for the offset and width, the scalar version packs
821     // the width and offset into a single operand. Try to move to the scalar
822     // version if the offsets are constant, so that we can try to keep extended
823     // loads of kernel arguments in SGPRs.
824 
825     // TODO: Technically we could try to pattern match scalar bitshifts of
826     // dynamic values, but it's probably not useful.
827     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
828     if (!Offset)
829       break;
830 
831     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
832     if (!Width)
833       break;
834 
835     bool Signed = Opc == AMDGPUISD::BFE_I32;
836 
837     uint32_t OffsetVal = Offset->getZExtValue();
838     uint32_t WidthVal = Width->getZExtValue();
839 
840     ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
841                             SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
842     return;
843   }
844   case AMDGPUISD::DIV_SCALE: {
845     SelectDIV_SCALE(N);
846     return;
847   }
848   case AMDGPUISD::MAD_I64_I32:
849   case AMDGPUISD::MAD_U64_U32: {
850     SelectMAD_64_32(N);
851     return;
852   }
853   case ISD::CopyToReg: {
854     const SITargetLowering& Lowering =
855       *static_cast<const SITargetLowering*>(getTargetLowering());
856     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
857     break;
858   }
859   case ISD::AND:
860   case ISD::SRL:
861   case ISD::SRA:
862   case ISD::SIGN_EXTEND_INREG:
863     if (N->getValueType(0) != MVT::i32)
864       break;
865 
866     SelectS_BFE(N);
867     return;
868   case ISD::BRCOND:
869     SelectBRCOND(N);
870     return;
871   case ISD::FMAD:
872   case ISD::FMA:
873     SelectFMAD_FMA(N);
874     return;
875   case AMDGPUISD::ATOMIC_CMP_SWAP:
876     SelectATOMIC_CMP_SWAP(N);
877     return;
878   case AMDGPUISD::CVT_PKRTZ_F16_F32:
879   case AMDGPUISD::CVT_PKNORM_I16_F32:
880   case AMDGPUISD::CVT_PKNORM_U16_F32:
881   case AMDGPUISD::CVT_PK_U16_U32:
882   case AMDGPUISD::CVT_PK_I16_I32: {
883     // Hack around using a legal type if f16 is illegal.
884     if (N->getValueType(0) == MVT::i32) {
885       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
886       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
887                               { N->getOperand(0), N->getOperand(1) });
888       SelectCode(N);
889       return;
890     }
891 
892     break;
893   }
894   case ISD::INTRINSIC_W_CHAIN: {
895     SelectINTRINSIC_W_CHAIN(N);
896     return;
897   }
898   case ISD::INTRINSIC_WO_CHAIN: {
899     SelectINTRINSIC_WO_CHAIN(N);
900     return;
901   }
902   case ISD::INTRINSIC_VOID: {
903     SelectINTRINSIC_VOID(N);
904     return;
905   }
906   }
907 
908   SelectCode(N);
909 }
910 
911 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
912   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
913   const Instruction *Term = BB->getTerminator();
914   return Term->getMetadata("amdgpu.uniform") ||
915          Term->getMetadata("structurizecfg.uniform");
916 }
917 
918 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
919                                           SDValue &N0, SDValue &N1) {
920   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
921       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
922     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
923     // (i64 (bitcast (v2i32 (build_vector
924     //                        (or (extract_vector_elt V, 0), OFFSET),
925     //                        (extract_vector_elt V, 1)))))
926     SDValue Lo = Addr.getOperand(0).getOperand(0);
927     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
928       SDValue BaseLo = Lo.getOperand(0);
929       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
930       // Check that split base (Lo and Hi) are extracted from the same one.
931       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
932           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
933           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
934           // Lo is statically extracted from index 0.
935           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
936           BaseLo.getConstantOperandVal(1) == 0 &&
937           // Hi is statically extracted from index 0.
938           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
939           BaseHi.getConstantOperandVal(1) == 1) {
940         N0 = BaseLo.getOperand(0).getOperand(0);
941         N1 = Lo.getOperand(1);
942         return true;
943       }
944     }
945   }
946   return false;
947 }
948 
949 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
950                                                     SDValue &RHS) const {
951   if (CurDAG->isBaseWithConstantOffset(Addr)) {
952     LHS = Addr.getOperand(0);
953     RHS = Addr.getOperand(1);
954     return true;
955   }
956 
957   if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) {
958     assert(LHS && RHS && isa<ConstantSDNode>(RHS));
959     return true;
960   }
961 
962   return false;
963 }
964 
965 StringRef AMDGPUDAGToDAGISel::getPassName() const {
966   return "AMDGPU DAG->DAG Pattern Instruction Selection";
967 }
968 
969 //===----------------------------------------------------------------------===//
970 // Complex Patterns
971 //===----------------------------------------------------------------------===//
972 
973 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
974                                             SDValue &Offset) {
975   return false;
976 }
977 
978 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
979                                             SDValue &Offset) {
980   ConstantSDNode *C;
981   SDLoc DL(Addr);
982 
983   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
984     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
985     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
986   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
987              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
988     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
989     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
990   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
991             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
992     Base = Addr.getOperand(0);
993     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
994   } else {
995     Base = Addr;
996     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
997   }
998 
999   return true;
1000 }
1001 
1002 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
1003                                                        const SDLoc &DL) const {
1004   SDNode *Mov = CurDAG->getMachineNode(
1005     AMDGPU::S_MOV_B32, DL, MVT::i32,
1006     CurDAG->getTargetConstant(Val, DL, MVT::i32));
1007   return SDValue(Mov, 0);
1008 }
1009 
1010 // FIXME: Should only handle addcarry/subcarry
1011 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
1012   SDLoc DL(N);
1013   SDValue LHS = N->getOperand(0);
1014   SDValue RHS = N->getOperand(1);
1015 
1016   unsigned Opcode = N->getOpcode();
1017   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
1018   bool ProduceCarry =
1019       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
1020   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
1021 
1022   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1023   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1024 
1025   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1026                                        DL, MVT::i32, LHS, Sub0);
1027   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1028                                        DL, MVT::i32, LHS, Sub1);
1029 
1030   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1031                                        DL, MVT::i32, RHS, Sub0);
1032   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1033                                        DL, MVT::i32, RHS, Sub1);
1034 
1035   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
1036 
1037   static const unsigned OpcMap[2][2][2] = {
1038       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
1039        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
1040       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
1041        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
1042 
1043   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
1044   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
1045 
1046   SDNode *AddLo;
1047   if (!ConsumeCarry) {
1048     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1049     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
1050   } else {
1051     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1052     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1053   }
1054   SDValue AddHiArgs[] = {
1055     SDValue(Hi0, 0),
1056     SDValue(Hi1, 0),
1057     SDValue(AddLo, 1)
1058   };
1059   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
1060 
1061   SDValue RegSequenceArgs[] = {
1062     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1063     SDValue(AddLo,0),
1064     Sub0,
1065     SDValue(AddHi,0),
1066     Sub1,
1067   };
1068   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1069                                                MVT::i64, RegSequenceArgs);
1070 
1071   if (ProduceCarry) {
1072     // Replace the carry-use
1073     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1074   }
1075 
1076   // Replace the remaining uses.
1077   ReplaceNode(N, RegSequence);
1078 }
1079 
1080 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1081   SDLoc DL(N);
1082   SDValue LHS = N->getOperand(0);
1083   SDValue RHS = N->getOperand(1);
1084   SDValue CI = N->getOperand(2);
1085 
1086   if (N->isDivergent()) {
1087     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
1088                                                    : AMDGPU::V_SUBB_U32_e64;
1089     CurDAG->SelectNodeTo(
1090         N, Opc, N->getVTList(),
1091         {LHS, RHS, CI,
1092          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1093   } else {
1094     unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
1095                                                    : AMDGPU::S_SUB_CO_PSEUDO;
1096     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
1097   }
1098 }
1099 
1100 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1101   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1102   // carry out despite the _i32 name. These were renamed in VI to _U32.
1103   // FIXME: We should probably rename the opcodes here.
1104   bool IsAdd = N->getOpcode() == ISD::UADDO;
1105   bool IsVALU = N->isDivergent();
1106 
1107   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
1108        ++UI)
1109     if (UI.getUse().getResNo() == 1) {
1110       if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
1111           (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
1112         IsVALU = true;
1113         break;
1114       }
1115     }
1116 
1117   if (IsVALU) {
1118     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1119 
1120     CurDAG->SelectNodeTo(
1121         N, Opc, N->getVTList(),
1122         {N->getOperand(0), N->getOperand(1),
1123          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1124   } else {
1125     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
1126                                                 : AMDGPU::S_USUBO_PSEUDO;
1127 
1128     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
1129                          {N->getOperand(0), N->getOperand(1)});
1130   }
1131 }
1132 
1133 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1134   SDLoc SL(N);
1135   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1136   SDValue Ops[10];
1137 
1138   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1139   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1140   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1141   Ops[8] = N->getOperand(0);
1142   Ops[9] = N->getOperand(4);
1143 
1144   CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32_e64, N->getVTList(), Ops);
1145 }
1146 
1147 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1148   SDLoc SL(N);
1149   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
1150   SDValue Ops[8];
1151 
1152   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1153   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1154   Ops[6] = N->getOperand(0);
1155   Ops[7] = N->getOperand(3);
1156 
1157   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1158 }
1159 
1160 // We need to handle this here because tablegen doesn't support matching
1161 // instructions with multiple outputs.
1162 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1163   SDLoc SL(N);
1164   EVT VT = N->getValueType(0);
1165 
1166   assert(VT == MVT::f32 || VT == MVT::f64);
1167 
1168   unsigned Opc
1169     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
1170 
1171   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
1172   // omod
1173   SDValue Ops[8];
1174   SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
1175   SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
1176   SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
1177   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1178 }
1179 
1180 // We need to handle this here because tablegen doesn't support matching
1181 // instructions with multiple outputs.
1182 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1183   SDLoc SL(N);
1184   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1185   unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1186 
1187   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1188   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1189                     Clamp };
1190   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1191 }
1192 
1193 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1194   if (!isUInt<16>(Offset))
1195     return false;
1196 
1197   if (!Base || Subtarget->hasUsableDSOffset() ||
1198       Subtarget->unsafeDSOffsetFoldingEnabled())
1199     return true;
1200 
1201   // On Southern Islands instruction with a negative base value and an offset
1202   // don't seem to work.
1203   return CurDAG->SignBitIsZero(Base);
1204 }
1205 
1206 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1207                                               SDValue &Offset) const {
1208   SDLoc DL(Addr);
1209   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1210     SDValue N0 = Addr.getOperand(0);
1211     SDValue N1 = Addr.getOperand(1);
1212     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1213     if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1214       // (add n0, c0)
1215       Base = N0;
1216       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1217       return true;
1218     }
1219   } else if (Addr.getOpcode() == ISD::SUB) {
1220     // sub C, x -> add (sub 0, x), C
1221     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1222       int64_t ByteOffset = C->getSExtValue();
1223       if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1224         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1225 
1226         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1227         // the known bits in isDSOffsetLegal. We need to emit the selected node
1228         // here, so this is thrown away.
1229         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1230                                       Zero, Addr.getOperand(1));
1231 
1232         if (isDSOffsetLegal(Sub, ByteOffset)) {
1233           SmallVector<SDValue, 3> Opnds;
1234           Opnds.push_back(Zero);
1235           Opnds.push_back(Addr.getOperand(1));
1236 
1237           // FIXME: Select to VOP3 version for with-carry.
1238           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1239           if (Subtarget->hasAddNoCarry()) {
1240             SubOp = AMDGPU::V_SUB_U32_e64;
1241             Opnds.push_back(
1242                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1243           }
1244 
1245           MachineSDNode *MachineSub =
1246               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1247 
1248           Base = SDValue(MachineSub, 0);
1249           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1250           return true;
1251         }
1252       }
1253     }
1254   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1255     // If we have a constant address, prefer to put the constant into the
1256     // offset. This can save moves to load the constant address since multiple
1257     // operations can share the zero base address register, and enables merging
1258     // into read2 / write2 instructions.
1259 
1260     SDLoc DL(Addr);
1261 
1262     if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1263       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1264       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1265                                  DL, MVT::i32, Zero);
1266       Base = SDValue(MovZero, 0);
1267       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1268       return true;
1269     }
1270   }
1271 
1272   // default case
1273   Base = Addr;
1274   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1275   return true;
1276 }
1277 
1278 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1279                                           unsigned Offset1,
1280                                           unsigned Size) const {
1281   if (Offset0 % Size != 0 || Offset1 % Size != 0)
1282     return false;
1283   if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1284     return false;
1285 
1286   if (!Base || Subtarget->hasUsableDSOffset() ||
1287       Subtarget->unsafeDSOffsetFoldingEnabled())
1288     return true;
1289 
1290   // On Southern Islands instruction with a negative base value and an offset
1291   // don't seem to work.
1292   return CurDAG->SignBitIsZero(Base);
1293 }
1294 
1295 // TODO: If offset is too big, put low 16-bit into offset.
1296 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1297                                                    SDValue &Offset0,
1298                                                    SDValue &Offset1) const {
1299   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1300 }
1301 
1302 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1303                                                     SDValue &Offset0,
1304                                                     SDValue &Offset1) const {
1305   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1306 }
1307 
1308 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1309                                             SDValue &Offset0, SDValue &Offset1,
1310                                             unsigned Size) const {
1311   SDLoc DL(Addr);
1312 
1313   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1314     SDValue N0 = Addr.getOperand(0);
1315     SDValue N1 = Addr.getOperand(1);
1316     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1317     unsigned OffsetValue0 = C1->getZExtValue();
1318     unsigned OffsetValue1 = OffsetValue0 + Size;
1319 
1320     // (add n0, c0)
1321     if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1322       Base = N0;
1323       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1324       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1325       return true;
1326     }
1327   } else if (Addr.getOpcode() == ISD::SUB) {
1328     // sub C, x -> add (sub 0, x), C
1329     if (const ConstantSDNode *C =
1330             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1331       unsigned OffsetValue0 = C->getZExtValue();
1332       unsigned OffsetValue1 = OffsetValue0 + Size;
1333 
1334       if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1335         SDLoc DL(Addr);
1336         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1337 
1338         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1339         // the known bits in isDSOffsetLegal. We need to emit the selected node
1340         // here, so this is thrown away.
1341         SDValue Sub =
1342             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1343 
1344         if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1345           SmallVector<SDValue, 3> Opnds;
1346           Opnds.push_back(Zero);
1347           Opnds.push_back(Addr.getOperand(1));
1348           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1349           if (Subtarget->hasAddNoCarry()) {
1350             SubOp = AMDGPU::V_SUB_U32_e64;
1351             Opnds.push_back(
1352                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1353           }
1354 
1355           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1356               SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1357 
1358           Base = SDValue(MachineSub, 0);
1359           Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1360           Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1361           return true;
1362         }
1363       }
1364     }
1365   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1366     unsigned OffsetValue0 = CAddr->getZExtValue();
1367     unsigned OffsetValue1 = OffsetValue0 + Size;
1368 
1369     if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1370       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1371       MachineSDNode *MovZero =
1372           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1373       Base = SDValue(MovZero, 0);
1374       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1375       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1376       return true;
1377     }
1378   }
1379 
1380   // default case
1381 
1382   Base = Addr;
1383   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1384   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1385   return true;
1386 }
1387 
1388 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1389                                      SDValue &SOffset, SDValue &Offset,
1390                                      SDValue &Offen, SDValue &Idxen,
1391                                      SDValue &Addr64) const {
1392   // Subtarget prefers to use flat instruction
1393   // FIXME: This should be a pattern predicate and not reach here
1394   if (Subtarget->useFlatForGlobal())
1395     return false;
1396 
1397   SDLoc DL(Addr);
1398 
1399   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1400   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1401   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1402   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1403 
1404   ConstantSDNode *C1 = nullptr;
1405   SDValue N0 = Addr;
1406   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1407     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1408     if (isUInt<32>(C1->getZExtValue()))
1409       N0 = Addr.getOperand(0);
1410     else
1411       C1 = nullptr;
1412   }
1413 
1414   if (N0.getOpcode() == ISD::ADD) {
1415     // (add N2, N3) -> addr64, or
1416     // (add (add N2, N3), C1) -> addr64
1417     SDValue N2 = N0.getOperand(0);
1418     SDValue N3 = N0.getOperand(1);
1419     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1420 
1421     if (N2->isDivergent()) {
1422       if (N3->isDivergent()) {
1423         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1424         // addr64, and construct the resource from a 0 address.
1425         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1426         VAddr = N0;
1427       } else {
1428         // N2 is divergent, N3 is not.
1429         Ptr = N3;
1430         VAddr = N2;
1431       }
1432     } else {
1433       // N2 is not divergent.
1434       Ptr = N2;
1435       VAddr = N3;
1436     }
1437     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1438   } else if (N0->isDivergent()) {
1439     // N0 is divergent. Use it as the addr64, and construct the resource from a
1440     // 0 address.
1441     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1442     VAddr = N0;
1443     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1444   } else {
1445     // N0 -> offset, or
1446     // (N0 + C1) -> offset
1447     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1448     Ptr = N0;
1449   }
1450 
1451   if (!C1) {
1452     // No offset.
1453     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1454     return true;
1455   }
1456 
1457   if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1458     // Legal offset for instruction.
1459     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1460     return true;
1461   }
1462 
1463   // Illegal offset, store it in soffset.
1464   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1465   SOffset =
1466       SDValue(CurDAG->getMachineNode(
1467                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1468                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1469               0);
1470   return true;
1471 }
1472 
1473 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1474                                            SDValue &VAddr, SDValue &SOffset,
1475                                            SDValue &Offset) const {
1476   SDValue Ptr, Offen, Idxen, Addr64;
1477 
1478   // addr64 bit was removed for volcanic islands.
1479   // FIXME: This should be a pattern predicate and not reach here
1480   if (!Subtarget->hasAddr64())
1481     return false;
1482 
1483   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1484     return false;
1485 
1486   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1487   if (C->getSExtValue()) {
1488     SDLoc DL(Addr);
1489 
1490     const SITargetLowering& Lowering =
1491       *static_cast<const SITargetLowering*>(getTargetLowering());
1492 
1493     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1494     return true;
1495   }
1496 
1497   return false;
1498 }
1499 
1500 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1501   SDLoc DL(N);
1502 
1503   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1504   SDValue TFI =
1505       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1506 
1507   // We rebase the base address into an absolute stack address and hence
1508   // use constant 0 for soffset. This value must be retained until
1509   // frame elimination and eliminateFrameIndex will choose the appropriate
1510   // frame register if need be.
1511   return std::make_pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1512 }
1513 
1514 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1515                                                  SDValue Addr, SDValue &Rsrc,
1516                                                  SDValue &VAddr, SDValue &SOffset,
1517                                                  SDValue &ImmOffset) const {
1518 
1519   SDLoc DL(Addr);
1520   MachineFunction &MF = CurDAG->getMachineFunction();
1521   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1522 
1523   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1524 
1525   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1526     int64_t Imm = CAddr->getSExtValue();
1527     const int64_t NullPtr =
1528         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1529     // Don't fold null pointer.
1530     if (Imm != NullPtr) {
1531       SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1532       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1533         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1534       VAddr = SDValue(MovHighBits, 0);
1535 
1536       SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1537       ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1538       return true;
1539     }
1540   }
1541 
1542   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1543     // (add n0, c1)
1544 
1545     SDValue N0 = Addr.getOperand(0);
1546     SDValue N1 = Addr.getOperand(1);
1547 
1548     // Offsets in vaddr must be positive if range checking is enabled.
1549     //
1550     // The total computation of vaddr + soffset + offset must not overflow.  If
1551     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1552     // overflowing.
1553     //
1554     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1555     // always perform a range check. If a negative vaddr base index was used,
1556     // this would fail the range check. The overall address computation would
1557     // compute a valid address, but this doesn't happen due to the range
1558     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1559     //
1560     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1561     // MUBUF vaddr, but not on older subtargets which can only do this if the
1562     // sign bit is known 0.
1563     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1564     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1565         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1566          CurDAG->SignBitIsZero(N0))) {
1567       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1568       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1569       return true;
1570     }
1571   }
1572 
1573   // (node)
1574   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1575   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1576   return true;
1577 }
1578 
1579 static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1580   if (Val.getOpcode() != ISD::CopyFromReg)
1581     return false;
1582   auto RC =
1583       TRI.getPhysRegClass(cast<RegisterSDNode>(Val.getOperand(1))->getReg());
1584   return RC && TRI.isSGPRClass(RC);
1585 }
1586 
1587 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1588                                                   SDValue Addr,
1589                                                   SDValue &SRsrc,
1590                                                   SDValue &SOffset,
1591                                                   SDValue &Offset) const {
1592   const SIRegisterInfo *TRI =
1593       static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
1594   MachineFunction &MF = CurDAG->getMachineFunction();
1595   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1596   SDLoc DL(Addr);
1597 
1598   // CopyFromReg <sgpr>
1599   if (IsCopyFromSGPR(*TRI, Addr)) {
1600     SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1601     SOffset = Addr;
1602     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1603     return true;
1604   }
1605 
1606   ConstantSDNode *CAddr;
1607   if (Addr.getOpcode() == ISD::ADD) {
1608     // Add (CopyFromReg <sgpr>) <constant>
1609     CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
1610     if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1611       return false;
1612     if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
1613       return false;
1614 
1615     SOffset = Addr.getOperand(0);
1616   } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
1617              SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
1618     // <constant>
1619     SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1620   } else {
1621     return false;
1622   }
1623 
1624   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1625 
1626   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1627   return true;
1628 }
1629 
1630 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1631                                            SDValue &SOffset, SDValue &Offset
1632                                            ) const {
1633   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1634   const SIInstrInfo *TII =
1635     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1636 
1637   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1638     return false;
1639 
1640   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1641       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1642       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1643     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1644                     APInt::getAllOnesValue(32).getZExtValue(); // Size
1645     SDLoc DL(Addr);
1646 
1647     const SITargetLowering& Lowering =
1648       *static_cast<const SITargetLowering*>(getTargetLowering());
1649 
1650     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1651     return true;
1652   }
1653   return false;
1654 }
1655 
1656 // Find a load or store from corresponding pattern root.
1657 // Roots may be build_vector, bitconvert or their combinations.
1658 static MemSDNode* findMemSDNode(SDNode *N) {
1659   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1660   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1661     return MN;
1662   assert(isa<BuildVectorSDNode>(N));
1663   for (SDValue V : N->op_values())
1664     if (MemSDNode *MN =
1665           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1666       return MN;
1667   llvm_unreachable("cannot find MemSDNode in the pattern!");
1668 }
1669 
1670 bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(SDNode *N, SDValue Addr,
1671                                               SDValue &VAddr, SDValue &Offset,
1672                                               uint64_t FlatVariant) const {
1673   int64_t OffsetVal = 0;
1674 
1675   unsigned AS = findMemSDNode(N)->getAddressSpace();
1676 
1677   bool CanHaveFlatSegmentOffsetBug =
1678       Subtarget->hasFlatSegmentOffsetBug() &&
1679       FlatVariant == SIInstrFlags::FLAT &&
1680       (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS);
1681 
1682   if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1683     SDValue N0, N1;
1684     if (isBaseWithConstantOffset64(Addr, N0, N1)) {
1685       int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1686 
1687       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1688       if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1689         Addr = N0;
1690         OffsetVal = COffsetVal;
1691       } else {
1692         // If the offset doesn't fit, put the low bits into the offset field and
1693         // add the rest.
1694         //
1695         // For a FLAT instruction the hardware decides whether to access
1696         // global/scratch/shared memory based on the high bits of vaddr,
1697         // ignoring the offset field, so we have to ensure that when we add
1698         // remainder to vaddr it still points into the same underlying object.
1699         // The easiest way to do that is to make sure that we split the offset
1700         // into two pieces that are both >= 0 or both <= 0.
1701 
1702         SDLoc DL(N);
1703         uint64_t RemainderOffset;
1704 
1705         std::tie(OffsetVal, RemainderOffset) =
1706             TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1707 
1708         SDValue AddOffsetLo =
1709             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1710         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1711 
1712         if (Addr.getValueType().getSizeInBits() == 32) {
1713           SmallVector<SDValue, 3> Opnds;
1714           Opnds.push_back(N0);
1715           Opnds.push_back(AddOffsetLo);
1716           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1717           if (Subtarget->hasAddNoCarry()) {
1718             AddOp = AMDGPU::V_ADD_U32_e64;
1719             Opnds.push_back(Clamp);
1720           }
1721           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1722         } else {
1723           // TODO: Should this try to use a scalar add pseudo if the base address
1724           // is uniform and saddr is usable?
1725           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1726           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1727 
1728           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1729                                                 DL, MVT::i32, N0, Sub0);
1730           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1731                                                 DL, MVT::i32, N0, Sub1);
1732 
1733           SDValue AddOffsetHi =
1734               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1735 
1736           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1737 
1738           SDNode *Add =
1739               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1740                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1741 
1742           SDNode *Addc = CurDAG->getMachineNode(
1743               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1744               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1745 
1746           SDValue RegSequenceArgs[] = {
1747               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1748               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1749 
1750           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1751                                                 MVT::i64, RegSequenceArgs),
1752                          0);
1753         }
1754       }
1755     }
1756   }
1757 
1758   VAddr = Addr;
1759   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1760   return true;
1761 }
1762 
1763 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
1764                                           SDValue &VAddr,
1765                                           SDValue &Offset) const {
1766   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FLAT);
1767 }
1768 
1769 bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
1770                                             SDValue &VAddr,
1771                                             SDValue &Offset) const {
1772   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FlatGlobal);
1773 }
1774 
1775 bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
1776                                              SDValue &VAddr,
1777                                              SDValue &Offset) const {
1778   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1779                               SIInstrFlags::FlatScratch);
1780 }
1781 
1782 // If this matches zero_extend i32:x, return x
1783 static SDValue matchZExtFromI32(SDValue Op) {
1784   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1785     return SDValue();
1786 
1787   SDValue ExtSrc = Op.getOperand(0);
1788   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1789 }
1790 
1791 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1792 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1793                                            SDValue Addr,
1794                                            SDValue &SAddr,
1795                                            SDValue &VOffset,
1796                                            SDValue &Offset) const {
1797   int64_t ImmOffset = 0;
1798 
1799   // Match the immediate offset first, which canonically is moved as low as
1800   // possible.
1801 
1802   SDValue LHS, RHS;
1803   if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1804     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1805     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1806 
1807     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS,
1808                                SIInstrFlags::FlatGlobal)) {
1809       Addr = LHS;
1810       ImmOffset = COffsetVal;
1811     } else if (!LHS->isDivergent()) {
1812       if (COffsetVal > 0) {
1813         SDLoc SL(N);
1814         // saddr + large_offset -> saddr +
1815         //                         (voffset = large_offset & ~MaxOffset) +
1816         //                         (large_offset & MaxOffset);
1817         int64_t SplitImmOffset, RemainderOffset;
1818         std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1819             COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal);
1820 
1821         if (isUInt<32>(RemainderOffset)) {
1822           SDNode *VMov = CurDAG->getMachineNode(
1823               AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1824               CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1825           VOffset = SDValue(VMov, 0);
1826           SAddr = LHS;
1827           Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1828           return true;
1829         }
1830       }
1831 
1832       // We are adding a 64 bit SGPR and a constant. If constant bus limit
1833       // is 1 we would need to perform 1 or 2 extra moves for each half of
1834       // the constant and it is better to do a scalar add and then issue a
1835       // single VALU instruction to materialize zero. Otherwise it is less
1836       // instructions to perform VALU adds with immediates or inline literals.
1837       unsigned NumLiterals =
1838           !TII->isInlineConstant(APInt(32, COffsetVal & 0xffffffff)) +
1839           !TII->isInlineConstant(APInt(32, COffsetVal >> 32));
1840       if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
1841         return false;
1842     }
1843   }
1844 
1845   // Match the variable offset.
1846   if (Addr.getOpcode() == ISD::ADD) {
1847     LHS = Addr.getOperand(0);
1848     RHS = Addr.getOperand(1);
1849 
1850     if (!LHS->isDivergent()) {
1851       // add (i64 sgpr), (zero_extend (i32 vgpr))
1852       if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1853         SAddr = LHS;
1854         VOffset = ZextRHS;
1855       }
1856     }
1857 
1858     if (!SAddr && !RHS->isDivergent()) {
1859       // add (zero_extend (i32 vgpr)), (i64 sgpr)
1860       if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1861         SAddr = RHS;
1862         VOffset = ZextLHS;
1863       }
1864     }
1865 
1866     if (SAddr) {
1867       Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1868       return true;
1869     }
1870   }
1871 
1872   if (Addr->isDivergent() || Addr.getOpcode() == ISD::UNDEF ||
1873       isa<ConstantSDNode>(Addr))
1874     return false;
1875 
1876   // It's cheaper to materialize a single 32-bit zero for vaddr than the two
1877   // moves required to copy a 64-bit SGPR to VGPR.
1878   SAddr = Addr;
1879   SDNode *VMov =
1880       CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
1881                              CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
1882   VOffset = SDValue(VMov, 0);
1883   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1884   return true;
1885 }
1886 
1887 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
1888   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1889     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1890   } else if (SAddr.getOpcode() == ISD::ADD &&
1891              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1892     // Materialize this into a scalar move for scalar address to avoid
1893     // readfirstlane.
1894     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1895     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1896                                               FI->getValueType(0));
1897     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, SDLoc(SAddr),
1898                                            MVT::i32, TFI, SAddr.getOperand(1)),
1899                     0);
1900   }
1901 
1902   return SAddr;
1903 }
1904 
1905 // Match (32-bit SGPR base) + sext(imm offset)
1906 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
1907                                             SDValue &SAddr,
1908                                             SDValue &Offset) const {
1909   if (Addr->isDivergent())
1910     return false;
1911 
1912   SDLoc DL(Addr);
1913 
1914   int64_t COffsetVal = 0;
1915 
1916   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1917     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1918     SAddr = Addr.getOperand(0);
1919   } else {
1920     SAddr = Addr;
1921   }
1922 
1923   SAddr = SelectSAddrFI(CurDAG, SAddr);
1924 
1925   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1926 
1927   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
1928                               SIInstrFlags::FlatScratch)) {
1929     int64_t SplitImmOffset, RemainderOffset;
1930     std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1931         COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
1932 
1933     COffsetVal = SplitImmOffset;
1934 
1935     SDValue AddOffset =
1936         SAddr.getOpcode() == ISD::TargetFrameIndex
1937             ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
1938             : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32);
1939     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, DL, MVT::i32,
1940                                            SAddr, AddOffset), 0);
1941   }
1942 
1943   Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
1944 
1945   return true;
1946 }
1947 
1948 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1949                                           SDValue &Offset, bool &Imm) const {
1950   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1951   if (!C) {
1952     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1953         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1954       Offset = ByteOffsetNode;
1955       Imm = false;
1956       return true;
1957     }
1958     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1959       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1960         Offset = ByteOffsetNode.getOperand(0);
1961         Imm = false;
1962         return true;
1963       }
1964     }
1965     return false;
1966   }
1967 
1968   SDLoc SL(ByteOffsetNode);
1969   // GFX9 and GFX10 have signed byte immediate offsets.
1970   int64_t ByteOffset = C->getSExtValue();
1971   Optional<int64_t> EncodedOffset =
1972       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false);
1973   if (EncodedOffset) {
1974     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1975     Imm = true;
1976     return true;
1977   }
1978 
1979   // SGPR and literal offsets are unsigned.
1980   if (ByteOffset < 0)
1981     return false;
1982 
1983   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1984   if (EncodedOffset) {
1985     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1986     return true;
1987   }
1988 
1989   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1990     return false;
1991 
1992   SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1993   Offset = SDValue(
1994       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1995 
1996   return true;
1997 }
1998 
1999 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
2000   if (Addr.getValueType() != MVT::i32)
2001     return Addr;
2002 
2003   // Zero-extend a 32-bit address.
2004   SDLoc SL(Addr);
2005 
2006   const MachineFunction &MF = CurDAG->getMachineFunction();
2007   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2008   unsigned AddrHiVal = Info->get32BitAddressHighBits();
2009   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
2010 
2011   const SDValue Ops[] = {
2012     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
2013     Addr,
2014     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2015     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
2016             0),
2017     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
2018   };
2019 
2020   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
2021                                         Ops), 0);
2022 }
2023 
2024 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
2025                                      SDValue &Offset, bool &Imm) const {
2026   SDLoc SL(Addr);
2027 
2028   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
2029   // wraparound, because s_load instructions perform the addition in 64 bits.
2030   if ((Addr.getValueType() != MVT::i32 ||
2031        Addr->getFlags().hasNoUnsignedWrap())) {
2032     SDValue N0, N1;
2033     // Extract the base and offset if possible.
2034     if (CurDAG->isBaseWithConstantOffset(Addr) ||
2035         Addr.getOpcode() == ISD::ADD) {
2036       N0 = Addr.getOperand(0);
2037       N1 = Addr.getOperand(1);
2038     } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2039       assert(N0 && N1 && isa<ConstantSDNode>(N1));
2040     }
2041     if (N0 && N1) {
2042       if (SelectSMRDOffset(N1, Offset, Imm)) {
2043         SBase = Expand32BitAddress(N0);
2044         return true;
2045       }
2046     }
2047   }
2048   SBase = Expand32BitAddress(Addr);
2049   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
2050   Imm = true;
2051   return true;
2052 }
2053 
2054 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2055                                        SDValue &Offset) const {
2056   bool Imm = false;
2057   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
2058 }
2059 
2060 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2061                                          SDValue &Offset) const {
2062 
2063   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2064 
2065   bool Imm = false;
2066   if (!SelectSMRD(Addr, SBase, Offset, Imm))
2067     return false;
2068 
2069   return !Imm && isa<ConstantSDNode>(Offset);
2070 }
2071 
2072 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2073                                         SDValue &Offset) const {
2074   bool Imm = false;
2075   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
2076          !isa<ConstantSDNode>(Offset);
2077 }
2078 
2079 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
2080                                              SDValue &Offset) const {
2081   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2082     // The immediate offset for S_BUFFER instructions is unsigned.
2083     if (auto Imm =
2084             AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) {
2085       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2086       return true;
2087     }
2088   }
2089 
2090   return false;
2091 }
2092 
2093 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
2094                                                SDValue &Offset) const {
2095   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2096 
2097   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
2098     if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget,
2099                                                          C->getZExtValue())) {
2100       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
2101       return true;
2102     }
2103   }
2104 
2105   return false;
2106 }
2107 
2108 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2109                                             SDValue &Base,
2110                                             SDValue &Offset) const {
2111   SDLoc DL(Index);
2112 
2113   if (CurDAG->isBaseWithConstantOffset(Index)) {
2114     SDValue N0 = Index.getOperand(0);
2115     SDValue N1 = Index.getOperand(1);
2116     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2117 
2118     // (add n0, c0)
2119     // Don't peel off the offset (c0) if doing so could possibly lead
2120     // the base (n0) to be negative.
2121     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2122     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2123         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2124       Base = N0;
2125       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2126       return true;
2127     }
2128   }
2129 
2130   if (isa<ConstantSDNode>(Index))
2131     return false;
2132 
2133   Base = Index;
2134   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2135   return true;
2136 }
2137 
2138 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
2139                                      SDValue Val, uint32_t Offset,
2140                                      uint32_t Width) {
2141   // Transformation function, pack the offset and width of a BFE into
2142   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2143   // source, bits [5:0] contain the offset and bits [22:16] the width.
2144   uint32_t PackedVal = Offset | (Width << 16);
2145   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2146 
2147   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2148 }
2149 
2150 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2151   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2152   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2153   // Predicate: 0 < b <= c < 32
2154 
2155   const SDValue &Shl = N->getOperand(0);
2156   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2157   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2158 
2159   if (B && C) {
2160     uint32_t BVal = B->getZExtValue();
2161     uint32_t CVal = C->getZExtValue();
2162 
2163     if (0 < BVal && BVal <= CVal && CVal < 32) {
2164       bool Signed = N->getOpcode() == ISD::SRA;
2165       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2166 
2167       ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2168                               32 - CVal));
2169       return;
2170     }
2171   }
2172   SelectCode(N);
2173 }
2174 
2175 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2176   switch (N->getOpcode()) {
2177   case ISD::AND:
2178     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2179       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2180       // Predicate: isMask(mask)
2181       const SDValue &Srl = N->getOperand(0);
2182       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2183       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2184 
2185       if (Shift && Mask) {
2186         uint32_t ShiftVal = Shift->getZExtValue();
2187         uint32_t MaskVal = Mask->getZExtValue();
2188 
2189         if (isMask_32(MaskVal)) {
2190           uint32_t WidthVal = countPopulation(MaskVal);
2191 
2192           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2193                                   Srl.getOperand(0), ShiftVal, WidthVal));
2194           return;
2195         }
2196       }
2197     }
2198     break;
2199   case ISD::SRL:
2200     if (N->getOperand(0).getOpcode() == ISD::AND) {
2201       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2202       // Predicate: isMask(mask >> b)
2203       const SDValue &And = N->getOperand(0);
2204       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2205       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2206 
2207       if (Shift && Mask) {
2208         uint32_t ShiftVal = Shift->getZExtValue();
2209         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2210 
2211         if (isMask_32(MaskVal)) {
2212           uint32_t WidthVal = countPopulation(MaskVal);
2213 
2214           ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
2215                                   And.getOperand(0), ShiftVal, WidthVal));
2216           return;
2217         }
2218       }
2219     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2220       SelectS_BFEFromShifts(N);
2221       return;
2222     }
2223     break;
2224   case ISD::SRA:
2225     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2226       SelectS_BFEFromShifts(N);
2227       return;
2228     }
2229     break;
2230 
2231   case ISD::SIGN_EXTEND_INREG: {
2232     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2233     SDValue Src = N->getOperand(0);
2234     if (Src.getOpcode() != ISD::SRL)
2235       break;
2236 
2237     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2238     if (!Amt)
2239       break;
2240 
2241     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2242     ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
2243                             Amt->getZExtValue(), Width));
2244     return;
2245   }
2246   }
2247 
2248   SelectCode(N);
2249 }
2250 
2251 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2252   assert(N->getOpcode() == ISD::BRCOND);
2253   if (!N->hasOneUse())
2254     return false;
2255 
2256   SDValue Cond = N->getOperand(1);
2257   if (Cond.getOpcode() == ISD::CopyToReg)
2258     Cond = Cond.getOperand(2);
2259 
2260   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2261     return false;
2262 
2263   MVT VT = Cond.getOperand(0).getSimpleValueType();
2264   if (VT == MVT::i32)
2265     return true;
2266 
2267   if (VT == MVT::i64) {
2268     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2269 
2270     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2271     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2272   }
2273 
2274   return false;
2275 }
2276 
2277 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2278   SDValue Cond = N->getOperand(1);
2279 
2280   if (Cond.isUndef()) {
2281     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2282                          N->getOperand(2), N->getOperand(0));
2283     return;
2284   }
2285 
2286   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2287   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2288 
2289   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2290   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2291   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2292   SDLoc SL(N);
2293 
2294   if (!UseSCCBr) {
2295     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2296     // analyzed what generates the vcc value, so we do not know whether vcc
2297     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2298     // disabled lanes.
2299     //
2300     // For the case that we select S_CBRANCH_SCC1 and it gets
2301     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2302     // SIInstrInfo::moveToVALU which inserts the S_AND).
2303     //
2304     // We could add an analysis of what generates the vcc value here and omit
2305     // the S_AND when is unnecessary. But it would be better to add a separate
2306     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2307     // catches both cases.
2308     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2309                                                          : AMDGPU::S_AND_B64,
2310                      SL, MVT::i1,
2311                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2312                                                         : AMDGPU::EXEC,
2313                                          MVT::i1),
2314                     Cond),
2315                    0);
2316   }
2317 
2318   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2319   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2320                        N->getOperand(2), // Basic Block
2321                        VCC.getValue(0));
2322 }
2323 
2324 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
2325   MVT VT = N->getSimpleValueType(0);
2326   bool IsFMA = N->getOpcode() == ISD::FMA;
2327   if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
2328                          !Subtarget->hasFmaMixInsts()) ||
2329       ((IsFMA && Subtarget->hasMadMixInsts()) ||
2330        (!IsFMA && Subtarget->hasFmaMixInsts()))) {
2331     SelectCode(N);
2332     return;
2333   }
2334 
2335   SDValue Src0 = N->getOperand(0);
2336   SDValue Src1 = N->getOperand(1);
2337   SDValue Src2 = N->getOperand(2);
2338   unsigned Src0Mods, Src1Mods, Src2Mods;
2339 
2340   // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
2341   // using the conversion from f16.
2342   bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
2343   bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
2344   bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
2345 
2346   assert((IsFMA || !Mode.allFP32Denormals()) &&
2347          "fmad selected with denormals enabled");
2348   // TODO: We can select this with f32 denormals enabled if all the sources are
2349   // converted from f16 (in which case fmad isn't legal).
2350 
2351   if (Sel0 || Sel1 || Sel2) {
2352     // For dummy operands.
2353     SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2354     SDValue Ops[] = {
2355       CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2356       CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2357       CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2358       CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2359       Zero, Zero
2360     };
2361 
2362     CurDAG->SelectNodeTo(N,
2363                          IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2364                          MVT::f32, Ops);
2365   } else {
2366     SelectCode(N);
2367   }
2368 }
2369 
2370 // This is here because there isn't a way to use the generated sub0_sub1 as the
2371 // subreg index to EXTRACT_SUBREG in tablegen.
2372 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2373   MemSDNode *Mem = cast<MemSDNode>(N);
2374   unsigned AS = Mem->getAddressSpace();
2375   if (AS == AMDGPUAS::FLAT_ADDRESS) {
2376     SelectCode(N);
2377     return;
2378   }
2379 
2380   MVT VT = N->getSimpleValueType(0);
2381   bool Is32 = (VT == MVT::i32);
2382   SDLoc SL(N);
2383 
2384   MachineSDNode *CmpSwap = nullptr;
2385   if (Subtarget->hasAddr64()) {
2386     SDValue SRsrc, VAddr, SOffset, Offset;
2387 
2388     if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset)) {
2389       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2390         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2391       SDValue CmpVal = Mem->getOperand(2);
2392       SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32);
2393 
2394       // XXX - Do we care about glue operands?
2395 
2396       SDValue Ops[] = {CmpVal, VAddr, SRsrc, SOffset, Offset, CPol,
2397                        Mem->getChain()};
2398 
2399       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2400     }
2401   }
2402 
2403   if (!CmpSwap) {
2404     SDValue SRsrc, SOffset, Offset;
2405     if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset)) {
2406       unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2407         AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2408 
2409       SDValue CmpVal = Mem->getOperand(2);
2410       SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32);
2411       SDValue Ops[] = {CmpVal, SRsrc, SOffset, Offset, CPol, Mem->getChain()};
2412 
2413       CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2414     }
2415   }
2416 
2417   if (!CmpSwap) {
2418     SelectCode(N);
2419     return;
2420   }
2421 
2422   MachineMemOperand *MMO = Mem->getMemOperand();
2423   CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2424 
2425   unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2426   SDValue Extract
2427     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2428 
2429   ReplaceUses(SDValue(N, 0), Extract);
2430   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2431   CurDAG->RemoveDeadNode(N);
2432 }
2433 
2434 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2435   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2436   // be copied to an SGPR with readfirstlane.
2437   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2438     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2439 
2440   SDValue Chain = N->getOperand(0);
2441   SDValue Ptr = N->getOperand(2);
2442   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2443   MachineMemOperand *MMO = M->getMemOperand();
2444   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2445 
2446   SDValue Offset;
2447   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2448     SDValue PtrBase = Ptr.getOperand(0);
2449     SDValue PtrOffset = Ptr.getOperand(1);
2450 
2451     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2452     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
2453       N = glueCopyToM0(N, PtrBase);
2454       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2455     }
2456   }
2457 
2458   if (!Offset) {
2459     N = glueCopyToM0(N, Ptr);
2460     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2461   }
2462 
2463   SDValue Ops[] = {
2464     Offset,
2465     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2466     Chain,
2467     N->getOperand(N->getNumOperands() - 1) // New glue
2468   };
2469 
2470   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2471   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2472 }
2473 
2474 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2475   switch (IntrID) {
2476   case Intrinsic::amdgcn_ds_gws_init:
2477     return AMDGPU::DS_GWS_INIT;
2478   case Intrinsic::amdgcn_ds_gws_barrier:
2479     return AMDGPU::DS_GWS_BARRIER;
2480   case Intrinsic::amdgcn_ds_gws_sema_v:
2481     return AMDGPU::DS_GWS_SEMA_V;
2482   case Intrinsic::amdgcn_ds_gws_sema_br:
2483     return AMDGPU::DS_GWS_SEMA_BR;
2484   case Intrinsic::amdgcn_ds_gws_sema_p:
2485     return AMDGPU::DS_GWS_SEMA_P;
2486   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2487     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2488   default:
2489     llvm_unreachable("not a gws intrinsic");
2490   }
2491 }
2492 
2493 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2494   if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2495       !Subtarget->hasGWSSemaReleaseAll()) {
2496     // Let this error.
2497     SelectCode(N);
2498     return;
2499   }
2500 
2501   // Chain, intrinsic ID, vsrc, offset
2502   const bool HasVSrc = N->getNumOperands() == 4;
2503   assert(HasVSrc || N->getNumOperands() == 3);
2504 
2505   SDLoc SL(N);
2506   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2507   int ImmOffset = 0;
2508   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2509   MachineMemOperand *MMO = M->getMemOperand();
2510 
2511   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2512   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2513 
2514   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2515   // offset field) % 64. Some versions of the programming guide omit the m0
2516   // part, or claim it's from offset 0.
2517   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2518     // If we have a constant offset, try to use the 0 in m0 as the base.
2519     // TODO: Look into changing the default m0 initialization value. If the
2520     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2521     // the immediate offset.
2522     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2523     ImmOffset = ConstOffset->getZExtValue();
2524   } else {
2525     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2526       ImmOffset = BaseOffset.getConstantOperandVal(1);
2527       BaseOffset = BaseOffset.getOperand(0);
2528     }
2529 
2530     // Prefer to do the shift in an SGPR since it should be possible to use m0
2531     // as the result directly. If it's already an SGPR, it will be eliminated
2532     // later.
2533     SDNode *SGPROffset
2534       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2535                                BaseOffset);
2536     // Shift to offset in m0
2537     SDNode *M0Base
2538       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2539                                SDValue(SGPROffset, 0),
2540                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2541     glueCopyToM0(N, SDValue(M0Base, 0));
2542   }
2543 
2544   SDValue Chain = N->getOperand(0);
2545   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2546 
2547   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2548   SmallVector<SDValue, 5> Ops;
2549   if (HasVSrc)
2550     Ops.push_back(N->getOperand(2));
2551   Ops.push_back(OffsetField);
2552   Ops.push_back(Chain);
2553 
2554   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2555   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2556 }
2557 
2558 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2559   if (Subtarget->getLDSBankCount() != 16) {
2560     // This is a single instruction with a pattern.
2561     SelectCode(N);
2562     return;
2563   }
2564 
2565   SDLoc DL(N);
2566 
2567   // This requires 2 instructions. It is possible to write a pattern to support
2568   // this, but the generated isel emitter doesn't correctly deal with multiple
2569   // output instructions using the same physical register input. The copy to m0
2570   // is incorrectly placed before the second instruction.
2571   //
2572   // TODO: Match source modifiers.
2573   //
2574   // def : Pat <
2575   //   (int_amdgcn_interp_p1_f16
2576   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2577   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2578   //                             (i1 timm:$high), M0),
2579   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2580   //       timm:$attrchan, 0,
2581   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2582   //   let Predicates = [has16BankLDS];
2583   // }
2584 
2585   // 16 bank LDS
2586   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2587                                       N->getOperand(5), SDValue());
2588 
2589   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2590 
2591   SDNode *InterpMov =
2592     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2593         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2594         N->getOperand(3),  // Attr
2595         N->getOperand(2),  // Attrchan
2596         ToM0.getValue(1) // In glue
2597   });
2598 
2599   SDNode *InterpP1LV =
2600     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2601         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2602         N->getOperand(1), // Src0
2603         N->getOperand(3), // Attr
2604         N->getOperand(2), // Attrchan
2605         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2606         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2607         N->getOperand(4), // high
2608         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2609         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2610         SDValue(InterpMov, 1)
2611   });
2612 
2613   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2614 }
2615 
2616 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2617   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2618   switch (IntrID) {
2619   case Intrinsic::amdgcn_ds_append:
2620   case Intrinsic::amdgcn_ds_consume: {
2621     if (N->getValueType(0) != MVT::i32)
2622       break;
2623     SelectDSAppendConsume(N, IntrID);
2624     return;
2625   }
2626   }
2627 
2628   SelectCode(N);
2629 }
2630 
2631 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2632   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2633   unsigned Opcode;
2634   switch (IntrID) {
2635   case Intrinsic::amdgcn_wqm:
2636     Opcode = AMDGPU::WQM;
2637     break;
2638   case Intrinsic::amdgcn_softwqm:
2639     Opcode = AMDGPU::SOFT_WQM;
2640     break;
2641   case Intrinsic::amdgcn_wwm:
2642   case Intrinsic::amdgcn_strict_wwm:
2643     Opcode = AMDGPU::STRICT_WWM;
2644     break;
2645   case Intrinsic::amdgcn_strict_wqm:
2646     Opcode = AMDGPU::STRICT_WQM;
2647     break;
2648   case Intrinsic::amdgcn_interp_p1_f16:
2649     SelectInterpP1F16(N);
2650     return;
2651   default:
2652     SelectCode(N);
2653     return;
2654   }
2655 
2656   SDValue Src = N->getOperand(1);
2657   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2658 }
2659 
2660 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2661   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2662   switch (IntrID) {
2663   case Intrinsic::amdgcn_ds_gws_init:
2664   case Intrinsic::amdgcn_ds_gws_barrier:
2665   case Intrinsic::amdgcn_ds_gws_sema_v:
2666   case Intrinsic::amdgcn_ds_gws_sema_br:
2667   case Intrinsic::amdgcn_ds_gws_sema_p:
2668   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2669     SelectDS_GWS(N, IntrID);
2670     return;
2671   default:
2672     break;
2673   }
2674 
2675   SelectCode(N);
2676 }
2677 
2678 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2679                                             unsigned &Mods,
2680                                             bool AllowAbs) const {
2681   Mods = 0;
2682   Src = In;
2683 
2684   if (Src.getOpcode() == ISD::FNEG) {
2685     Mods |= SISrcMods::NEG;
2686     Src = Src.getOperand(0);
2687   }
2688 
2689   if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2690     Mods |= SISrcMods::ABS;
2691     Src = Src.getOperand(0);
2692   }
2693 
2694   return true;
2695 }
2696 
2697 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2698                                         SDValue &SrcMods) const {
2699   unsigned Mods;
2700   if (SelectVOP3ModsImpl(In, Src, Mods)) {
2701     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2702     return true;
2703   }
2704 
2705   return false;
2706 }
2707 
2708 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2709                                          SDValue &SrcMods) const {
2710   unsigned Mods;
2711   if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) {
2712     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2713     return true;
2714   }
2715 
2716   return false;
2717 }
2718 
2719 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2720                                              SDValue &SrcMods) const {
2721   SelectVOP3Mods(In, Src, SrcMods);
2722   return isNoNanSrc(Src);
2723 }
2724 
2725 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2726   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2727     return false;
2728 
2729   Src = In;
2730   return true;
2731 }
2732 
2733 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2734                                          SDValue &SrcMods, SDValue &Clamp,
2735                                          SDValue &Omod) const {
2736   SDLoc DL(In);
2737   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2738   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2739 
2740   return SelectVOP3Mods(In, Src, SrcMods);
2741 }
2742 
2743 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2744                                           SDValue &SrcMods, SDValue &Clamp,
2745                                           SDValue &Omod) const {
2746   SDLoc DL(In);
2747   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2748   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2749 
2750   return SelectVOP3BMods(In, Src, SrcMods);
2751 }
2752 
2753 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2754                                          SDValue &Clamp, SDValue &Omod) const {
2755   Src = In;
2756 
2757   SDLoc DL(In);
2758   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2759   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2760 
2761   return true;
2762 }
2763 
2764 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2765                                          SDValue &SrcMods) const {
2766   unsigned Mods = 0;
2767   Src = In;
2768 
2769   if (Src.getOpcode() == ISD::FNEG) {
2770     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2771     Src = Src.getOperand(0);
2772   }
2773 
2774   if (Src.getOpcode() == ISD::BUILD_VECTOR) {
2775     unsigned VecMods = Mods;
2776 
2777     SDValue Lo = stripBitcast(Src.getOperand(0));
2778     SDValue Hi = stripBitcast(Src.getOperand(1));
2779 
2780     if (Lo.getOpcode() == ISD::FNEG) {
2781       Lo = stripBitcast(Lo.getOperand(0));
2782       Mods ^= SISrcMods::NEG;
2783     }
2784 
2785     if (Hi.getOpcode() == ISD::FNEG) {
2786       Hi = stripBitcast(Hi.getOperand(0));
2787       Mods ^= SISrcMods::NEG_HI;
2788     }
2789 
2790     if (isExtractHiElt(Lo, Lo))
2791       Mods |= SISrcMods::OP_SEL_0;
2792 
2793     if (isExtractHiElt(Hi, Hi))
2794       Mods |= SISrcMods::OP_SEL_1;
2795 
2796     unsigned VecSize = Src.getValueSizeInBits();
2797     Lo = stripExtractLoElt(Lo);
2798     Hi = stripExtractLoElt(Hi);
2799 
2800     if (Lo.getValueSizeInBits() > VecSize) {
2801       Lo = CurDAG->getTargetExtractSubreg(
2802         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2803         MVT::getIntegerVT(VecSize), Lo);
2804     }
2805 
2806     if (Hi.getValueSizeInBits() > VecSize) {
2807       Hi = CurDAG->getTargetExtractSubreg(
2808         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2809         MVT::getIntegerVT(VecSize), Hi);
2810     }
2811 
2812     assert(Lo.getValueSizeInBits() <= VecSize &&
2813            Hi.getValueSizeInBits() <= VecSize);
2814 
2815     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2816       // Really a scalar input. Just select from the low half of the register to
2817       // avoid packing.
2818 
2819       if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
2820         Src = Lo;
2821       } else {
2822         assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
2823 
2824         SDLoc SL(In);
2825         SDValue Undef = SDValue(
2826           CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
2827                                  Lo.getValueType()), 0);
2828         auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID
2829                                     : AMDGPU::SReg_64RegClassID;
2830         const SDValue Ops[] = {
2831           CurDAG->getTargetConstant(RC, SL, MVT::i32),
2832           Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2833           Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) };
2834 
2835         Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
2836                                              Src.getValueType(), Ops), 0);
2837       }
2838       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2839       return true;
2840     }
2841 
2842     if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
2843       uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
2844                       .bitcastToAPInt().getZExtValue();
2845       if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
2846         Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);;
2847         SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2848         return true;
2849       }
2850     }
2851 
2852     Mods = VecMods;
2853   }
2854 
2855   // Packed instructions do not have abs modifiers.
2856   Mods |= SISrcMods::OP_SEL_1;
2857 
2858   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2859   return true;
2860 }
2861 
2862 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2863                                          SDValue &SrcMods) const {
2864   Src = In;
2865   // FIXME: Handle op_sel
2866   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2867   return true;
2868 }
2869 
2870 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2871                                              SDValue &SrcMods) const {
2872   // FIXME: Handle op_sel
2873   return SelectVOP3Mods(In, Src, SrcMods);
2874 }
2875 
2876 // The return value is not whether the match is possible (which it always is),
2877 // but whether or not it a conversion is really used.
2878 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2879                                                    unsigned &Mods) const {
2880   Mods = 0;
2881   SelectVOP3ModsImpl(In, Src, Mods);
2882 
2883   if (Src.getOpcode() == ISD::FP_EXTEND) {
2884     Src = Src.getOperand(0);
2885     assert(Src.getValueType() == MVT::f16);
2886     Src = stripBitcast(Src);
2887 
2888     // Be careful about folding modifiers if we already have an abs. fneg is
2889     // applied last, so we don't want to apply an earlier fneg.
2890     if ((Mods & SISrcMods::ABS) == 0) {
2891       unsigned ModsTmp;
2892       SelectVOP3ModsImpl(Src, Src, ModsTmp);
2893 
2894       if ((ModsTmp & SISrcMods::NEG) != 0)
2895         Mods ^= SISrcMods::NEG;
2896 
2897       if ((ModsTmp & SISrcMods::ABS) != 0)
2898         Mods |= SISrcMods::ABS;
2899     }
2900 
2901     // op_sel/op_sel_hi decide the source type and source.
2902     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2903     // If the sources's op_sel is set, it picks the high half of the source
2904     // register.
2905 
2906     Mods |= SISrcMods::OP_SEL_1;
2907     if (isExtractHiElt(Src, Src)) {
2908       Mods |= SISrcMods::OP_SEL_0;
2909 
2910       // TODO: Should we try to look for neg/abs here?
2911     }
2912 
2913     return true;
2914   }
2915 
2916   return false;
2917 }
2918 
2919 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2920                                                SDValue &SrcMods) const {
2921   unsigned Mods = 0;
2922   SelectVOP3PMadMixModsImpl(In, Src, Mods);
2923   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2924   return true;
2925 }
2926 
2927 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2928   if (In.isUndef())
2929     return CurDAG->getUNDEF(MVT::i32);
2930 
2931   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2932     SDLoc SL(In);
2933     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2934   }
2935 
2936   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2937     SDLoc SL(In);
2938     return CurDAG->getConstant(
2939       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2940   }
2941 
2942   SDValue Src;
2943   if (isExtractHiElt(In, Src))
2944     return Src;
2945 
2946   return SDValue();
2947 }
2948 
2949 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2950   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2951 
2952   const SIRegisterInfo *SIRI =
2953     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2954   const SIInstrInfo * SII =
2955     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2956 
2957   unsigned Limit = 0;
2958   bool AllUsesAcceptSReg = true;
2959   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2960     Limit < 10 && U != E; ++U, ++Limit) {
2961     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2962 
2963     // If the register class is unknown, it could be an unknown
2964     // register class that needs to be an SGPR, e.g. an inline asm
2965     // constraint
2966     if (!RC || SIRI->isSGPRClass(RC))
2967       return false;
2968 
2969     if (RC != &AMDGPU::VS_32RegClass) {
2970       AllUsesAcceptSReg = false;
2971       SDNode * User = *U;
2972       if (User->isMachineOpcode()) {
2973         unsigned Opc = User->getMachineOpcode();
2974         MCInstrDesc Desc = SII->get(Opc);
2975         if (Desc.isCommutable()) {
2976           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2977           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2978           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2979             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2980             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2981             if (CommutedRC == &AMDGPU::VS_32RegClass)
2982               AllUsesAcceptSReg = true;
2983           }
2984         }
2985       }
2986       // If "AllUsesAcceptSReg == false" so far we haven't suceeded
2987       // commuting current user. This means have at least one use
2988       // that strictly require VGPR. Thus, we will not attempt to commute
2989       // other user instructions.
2990       if (!AllUsesAcceptSReg)
2991         break;
2992     }
2993   }
2994   return !AllUsesAcceptSReg && (Limit < 10);
2995 }
2996 
2997 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2998   auto Ld = cast<LoadSDNode>(N);
2999 
3000   return Ld->getAlignment() >= 4 &&
3001         (
3002           (
3003             (
3004               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS       ||
3005               Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT
3006             )
3007             &&
3008             !N->isDivergent()
3009           )
3010           ||
3011           (
3012             Subtarget->getScalarizeGlobalBehavior() &&
3013             Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
3014             Ld->isSimple() &&
3015             !N->isDivergent() &&
3016             static_cast<const SITargetLowering *>(
3017               getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)
3018           )
3019         );
3020 }
3021 
3022 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
3023   const AMDGPUTargetLowering& Lowering =
3024     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
3025   bool IsModified = false;
3026   do {
3027     IsModified = false;
3028 
3029     // Go over all selected nodes and try to fold them a bit more
3030     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
3031     while (Position != CurDAG->allnodes_end()) {
3032       SDNode *Node = &*Position++;
3033       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
3034       if (!MachineNode)
3035         continue;
3036 
3037       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
3038       if (ResNode != Node) {
3039         if (ResNode)
3040           ReplaceUses(Node, ResNode);
3041         IsModified = true;
3042       }
3043     }
3044     CurDAG->RemoveDeadNodes();
3045   } while (IsModified);
3046 }
3047 
3048 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
3049   Subtarget = &MF.getSubtarget<R600Subtarget>();
3050   return SelectionDAGISel::runOnMachineFunction(MF);
3051 }
3052 
3053 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
3054   if (!N->readMem())
3055     return false;
3056   if (CbId == -1)
3057     return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
3058            N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
3059 
3060   return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
3061 }
3062 
3063 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
3064                                                          SDValue& IntPtr) {
3065   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
3066     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
3067                                        true);
3068     return true;
3069   }
3070   return false;
3071 }
3072 
3073 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
3074     SDValue& BaseReg, SDValue &Offset) {
3075   if (!isa<ConstantSDNode>(Addr)) {
3076     BaseReg = Addr;
3077     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
3078     return true;
3079   }
3080   return false;
3081 }
3082 
3083 void R600DAGToDAGISel::Select(SDNode *N) {
3084   unsigned int Opc = N->getOpcode();
3085   if (N->isMachineOpcode()) {
3086     N->setNodeId(-1);
3087     return;   // Already selected.
3088   }
3089 
3090   switch (Opc) {
3091   default: break;
3092   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
3093   case ISD::SCALAR_TO_VECTOR:
3094   case ISD::BUILD_VECTOR: {
3095     EVT VT = N->getValueType(0);
3096     unsigned NumVectorElts = VT.getVectorNumElements();
3097     unsigned RegClassID;
3098     // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
3099     // that adds a 128 bits reg copy when going through TwoAddressInstructions
3100     // pass. We want to avoid 128 bits copies as much as possible because they
3101     // can't be bundled by our scheduler.
3102     switch(NumVectorElts) {
3103     case 2: RegClassID = R600::R600_Reg64RegClassID; break;
3104     case 4:
3105       if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
3106         RegClassID = R600::R600_Reg128VerticalRegClassID;
3107       else
3108         RegClassID = R600::R600_Reg128RegClassID;
3109       break;
3110     default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
3111     }
3112     SelectBuildVector(N, RegClassID);
3113     return;
3114   }
3115   }
3116 
3117   SelectCode(N);
3118 }
3119 
3120 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
3121                                           SDValue &Offset) {
3122   ConstantSDNode *C;
3123   SDLoc DL(Addr);
3124 
3125   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
3126     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3127     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3128   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
3129              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
3130     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
3131     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3132   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
3133             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
3134     Base = Addr.getOperand(0);
3135     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
3136   } else {
3137     Base = Addr;
3138     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
3139   }
3140 
3141   return true;
3142 }
3143 
3144 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
3145                                           SDValue &Offset) {
3146   ConstantSDNode *IMMOffset;
3147 
3148   if (Addr.getOpcode() == ISD::ADD
3149       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
3150       && isInt<16>(IMMOffset->getZExtValue())) {
3151 
3152       Base = Addr.getOperand(0);
3153       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3154                                          MVT::i32);
3155       return true;
3156   // If the pointer address is constant, we can move it to the offset field.
3157   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
3158              && isInt<16>(IMMOffset->getZExtValue())) {
3159     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
3160                                   SDLoc(CurDAG->getEntryNode()),
3161                                   R600::ZERO, MVT::i32);
3162     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
3163                                        MVT::i32);
3164     return true;
3165   }
3166 
3167   // Default case, no offset
3168   Base = Addr;
3169   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
3170   return true;
3171 }
3172