1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //==-----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Defines an instruction selector for the AMDGPU target.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUInstrInfo.h"
16 #include "AMDGPUIntrinsicInfo.h"
17 #include "AMDGPUISelLowering.h" // For AMDGPUISD
18 #include "AMDGPUSubtarget.h"
19 #include "SIISelLowering.h"
20 #include "SIMachineFunctionInfo.h"
21 #include "llvm/CodeGen/FunctionLoweringInfo.h"
22 #include "llvm/CodeGen/PseudoSourceValue.h"
23 #include "llvm/CodeGen/SelectionDAG.h"
24 #include "llvm/CodeGen/SelectionDAGISel.h"
25 #include "llvm/IR/DiagnosticInfo.h"
26 
27 using namespace llvm;
28 
29 namespace llvm {
30 class R600InstrInfo;
31 }
32 
33 //===----------------------------------------------------------------------===//
34 // Instruction Selector Implementation
35 //===----------------------------------------------------------------------===//
36 
37 namespace {
38 
39 static bool isCBranchSCC(const SDNode *N) {
40   assert(N->getOpcode() == ISD::BRCOND);
41   if (!N->hasOneUse())
42     return false;
43 
44   SDValue Cond = N->getOperand(1);
45   if (Cond.getOpcode() == ISD::CopyToReg)
46     Cond = Cond.getOperand(2);
47   return Cond.getOpcode() == ISD::SETCC &&
48          Cond.getOperand(0).getValueType() == MVT::i32 &&
49 	 Cond.hasOneUse();
50 }
51 
52 /// AMDGPU specific code to select AMDGPU machine instructions for
53 /// SelectionDAG operations.
54 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
55   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
56   // make the right decision when generating code for different targets.
57   const AMDGPUSubtarget *Subtarget;
58 
59 public:
60   AMDGPUDAGToDAGISel(TargetMachine &TM);
61   virtual ~AMDGPUDAGToDAGISel();
62   bool runOnMachineFunction(MachineFunction &MF) override;
63   SDNode *Select(SDNode *N) override;
64   const char *getPassName() const override;
65   void PreprocessISelDAG() override;
66   void PostprocessISelDAG() override;
67 
68 private:
69   bool isInlineImmediate(SDNode *N) const;
70   bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
71                    const R600InstrInfo *TII);
72   bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
73   bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
74 
75   // Complex pattern selectors
76   bool SelectADDRParam(SDValue Addr, SDValue& R1, SDValue& R2);
77   bool SelectADDR(SDValue N, SDValue &R1, SDValue &R2);
78   bool SelectADDR64(SDValue N, SDValue &R1, SDValue &R2);
79 
80   static bool checkType(const Value *ptr, unsigned int addrspace);
81   static bool checkPrivateAddress(const MachineMemOperand *Op);
82 
83   static bool isGlobalStore(const StoreSDNode *N);
84   static bool isFlatStore(const StoreSDNode *N);
85   static bool isPrivateStore(const StoreSDNode *N);
86   static bool isLocalStore(const StoreSDNode *N);
87   static bool isRegionStore(const StoreSDNode *N);
88 
89   bool isCPLoad(const LoadSDNode *N) const;
90   bool isConstantLoad(const LoadSDNode *N, int cbID) const;
91   bool isGlobalLoad(const LoadSDNode *N) const;
92   bool isFlatLoad(const LoadSDNode *N) const;
93   bool isParamLoad(const LoadSDNode *N) const;
94   bool isPrivateLoad(const LoadSDNode *N) const;
95   bool isLocalLoad(const LoadSDNode *N) const;
96   bool isRegionLoad(const LoadSDNode *N) const;
97 
98   bool isUniformBr(const SDNode *N) const;
99 
100   SDNode *glueCopyToM0(SDNode *N) const;
101 
102   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
103   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
104   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
105                                        SDValue& Offset);
106   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
107   bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
108   bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
109                        unsigned OffsetBits) const;
110   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
111   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
112                                  SDValue &Offset1) const;
113   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
114                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
115                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
116                    SDValue &TFE) const;
117   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
118                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
119                          SDValue &SLC, SDValue &TFE) const;
120   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
121                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
122                          SDValue &SLC) const;
123   bool SelectMUBUFScratch(SDValue Addr, SDValue &RSrc, SDValue &VAddr,
124                           SDValue &SOffset, SDValue &ImmOffset) const;
125   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
126                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
127                          SDValue &TFE) const;
128   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
129                          SDValue &Offset, SDValue &GLC) const;
130   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
131                         bool &Imm) const;
132   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
133                   bool &Imm) const;
134   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
135   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
136   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
137   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
138   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
139   bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
140   SDNode *SelectAddrSpaceCast(SDNode *N);
141   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
142   bool SelectVOP3NoMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
143   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
144                        SDValue &Clamp, SDValue &Omod) const;
145   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
146                          SDValue &Clamp, SDValue &Omod) const;
147 
148   bool SelectVOP3Mods0Clamp(SDValue In, SDValue &Src, SDValue &SrcMods,
149                             SDValue &Omod) const;
150   bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
151                                  SDValue &Clamp,
152                                  SDValue &Omod) const;
153 
154   SDNode *SelectADD_SUB_I64(SDNode *N);
155   SDNode *SelectDIV_SCALE(SDNode *N);
156 
157   SDNode *getS_BFE(unsigned Opcode, SDLoc DL, SDValue Val,
158                    uint32_t Offset, uint32_t Width);
159   SDNode *SelectS_BFEFromShifts(SDNode *N);
160   SDNode *SelectS_BFE(SDNode *N);
161   SDNode *SelectBRCOND(SDNode *N);
162 
163   // Include the pieces autogenerated from the target description.
164 #include "AMDGPUGenDAGISel.inc"
165 };
166 }  // end anonymous namespace
167 
168 /// \brief This pass converts a legalized DAG into a AMDGPU-specific
169 // DAG, ready for instruction scheduling.
170 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM) {
171   return new AMDGPUDAGToDAGISel(TM);
172 }
173 
174 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM)
175     : SelectionDAGISel(TM) {}
176 
177 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
178   Subtarget = &static_cast<const AMDGPUSubtarget &>(MF.getSubtarget());
179   return SelectionDAGISel::runOnMachineFunction(MF);
180 }
181 
182 AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() {
183 }
184 
185 bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const {
186   const SITargetLowering *TL
187       = static_cast<const SITargetLowering *>(getTargetLowering());
188   return TL->analyzeImmediate(N) == 0;
189 }
190 
191 /// \brief Determine the register class for \p OpNo
192 /// \returns The register class of the virtual register that will be used for
193 /// the given operand number \OpNo or NULL if the register class cannot be
194 /// determined.
195 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
196                                                           unsigned OpNo) const {
197   if (!N->isMachineOpcode())
198     return nullptr;
199 
200   switch (N->getMachineOpcode()) {
201   default: {
202     const MCInstrDesc &Desc =
203         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
204     unsigned OpIdx = Desc.getNumDefs() + OpNo;
205     if (OpIdx >= Desc.getNumOperands())
206       return nullptr;
207     int RegClass = Desc.OpInfo[OpIdx].RegClass;
208     if (RegClass == -1)
209       return nullptr;
210 
211     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
212   }
213   case AMDGPU::REG_SEQUENCE: {
214     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
215     const TargetRegisterClass *SuperRC =
216         Subtarget->getRegisterInfo()->getRegClass(RCID);
217 
218     SDValue SubRegOp = N->getOperand(OpNo + 1);
219     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
220     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
221                                                               SubRegIdx);
222   }
223   }
224 }
225 
226 bool AMDGPUDAGToDAGISel::SelectADDRParam(
227   SDValue Addr, SDValue& R1, SDValue& R2) {
228 
229   if (Addr.getOpcode() == ISD::FrameIndex) {
230     if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
231       R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
232       R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
233     } else {
234       R1 = Addr;
235       R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
236     }
237   } else if (Addr.getOpcode() == ISD::ADD) {
238     R1 = Addr.getOperand(0);
239     R2 = Addr.getOperand(1);
240   } else {
241     R1 = Addr;
242     R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
243   }
244   return true;
245 }
246 
247 bool AMDGPUDAGToDAGISel::SelectADDR(SDValue Addr, SDValue& R1, SDValue& R2) {
248   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
249       Addr.getOpcode() == ISD::TargetGlobalAddress) {
250     return false;
251   }
252   return SelectADDRParam(Addr, R1, R2);
253 }
254 
255 
256 bool AMDGPUDAGToDAGISel::SelectADDR64(SDValue Addr, SDValue& R1, SDValue& R2) {
257   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
258       Addr.getOpcode() == ISD::TargetGlobalAddress) {
259     return false;
260   }
261 
262   if (Addr.getOpcode() == ISD::FrameIndex) {
263     if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
264       R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64);
265       R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
266     } else {
267       R1 = Addr;
268       R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
269     }
270   } else if (Addr.getOpcode() == ISD::ADD) {
271     R1 = Addr.getOperand(0);
272     R2 = Addr.getOperand(1);
273   } else {
274     R1 = Addr;
275     R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
276   }
277   return true;
278 }
279 
280 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
281   if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
282       !checkType(cast<MemSDNode>(N)->getMemOperand()->getValue(),
283                  AMDGPUAS::LOCAL_ADDRESS))
284     return N;
285 
286   const SITargetLowering& Lowering =
287       *static_cast<const SITargetLowering*>(getTargetLowering());
288 
289   // Write max value to m0 before each load operation
290 
291   SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
292                                  CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
293 
294   SDValue Glue = M0.getValue(1);
295 
296   SmallVector <SDValue, 8> Ops;
297   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
298      Ops.push_back(N->getOperand(i));
299   }
300   Ops.push_back(Glue);
301   CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
302 
303   return N;
304 }
305 
306 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
307   switch (NumVectorElts) {
308   case 1:
309     return AMDGPU::SReg_32RegClassID;
310   case 2:
311     return AMDGPU::SReg_64RegClassID;
312   case 4:
313     return AMDGPU::SReg_128RegClassID;
314   case 8:
315     return AMDGPU::SReg_256RegClassID;
316   case 16:
317     return AMDGPU::SReg_512RegClassID;
318   }
319 
320   llvm_unreachable("invalid vector size");
321 }
322 
323 SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
324   unsigned int Opc = N->getOpcode();
325   if (N->isMachineOpcode()) {
326     N->setNodeId(-1);
327     return nullptr;   // Already selected.
328   }
329 
330   if (isa<AtomicSDNode>(N))
331     N = glueCopyToM0(N);
332 
333   switch (Opc) {
334   default: break;
335   // We are selecting i64 ADD here instead of custom lower it during
336   // DAG legalization, so we can fold some i64 ADDs used for address
337   // calculation into the LOAD and STORE instructions.
338   case ISD::ADD:
339   case ISD::SUB: {
340     if (N->getValueType(0) != MVT::i64 ||
341         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
342       break;
343 
344     return SelectADD_SUB_I64(N);
345   }
346   case ISD::SCALAR_TO_VECTOR:
347   case AMDGPUISD::BUILD_VERTICAL_VECTOR:
348   case ISD::BUILD_VECTOR: {
349     unsigned RegClassID;
350     const AMDGPURegisterInfo *TRI = Subtarget->getRegisterInfo();
351     EVT VT = N->getValueType(0);
352     unsigned NumVectorElts = VT.getVectorNumElements();
353     EVT EltVT = VT.getVectorElementType();
354     assert(EltVT.bitsEq(MVT::i32));
355     if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
356       RegClassID = selectSGPRVectorRegClassID(NumVectorElts);
357     } else {
358       // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
359       // that adds a 128 bits reg copy when going through TwoAddressInstructions
360       // pass. We want to avoid 128 bits copies as much as possible because they
361       // can't be bundled by our scheduler.
362       switch(NumVectorElts) {
363       case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
364       case 4:
365         if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
366           RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
367         else
368           RegClassID = AMDGPU::R600_Reg128RegClassID;
369         break;
370       default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
371       }
372     }
373 
374     SDLoc DL(N);
375     SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
376 
377     if (NumVectorElts == 1) {
378       return CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT,
379                                   N->getOperand(0), RegClass);
380     }
381 
382     assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not "
383                                   "supported yet");
384     // 16 = Max Num Vector Elements
385     // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
386     // 1 = Vector Register Class
387     SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
388 
389     RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
390     bool IsRegSeq = true;
391     unsigned NOps = N->getNumOperands();
392     for (unsigned i = 0; i < NOps; i++) {
393       // XXX: Why is this here?
394       if (isa<RegisterSDNode>(N->getOperand(i))) {
395         IsRegSeq = false;
396         break;
397       }
398       RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
399       RegSeqArgs[1 + (2 * i) + 1] =
400               CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
401                                         MVT::i32);
402     }
403 
404     if (NOps != NumVectorElts) {
405       // Fill in the missing undef elements if this was a scalar_to_vector.
406       assert(Opc == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
407 
408       MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
409                                                      DL, EltVT);
410       for (unsigned i = NOps; i < NumVectorElts; ++i) {
411         RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
412         RegSeqArgs[1 + (2 * i) + 1] =
413           CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
414       }
415     }
416 
417     if (!IsRegSeq)
418       break;
419     return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
420                                 RegSeqArgs);
421   }
422   case ISD::BUILD_PAIR: {
423     SDValue RC, SubReg0, SubReg1;
424     if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
425       break;
426     }
427     SDLoc DL(N);
428     if (N->getValueType(0) == MVT::i128) {
429       RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
430       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
431       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
432     } else if (N->getValueType(0) == MVT::i64) {
433       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
434       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
435       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
436     } else {
437       llvm_unreachable("Unhandled value type for BUILD_PAIR");
438     }
439     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
440                             N->getOperand(1), SubReg1 };
441     return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
442                                   DL, N->getValueType(0), Ops);
443   }
444 
445   case ISD::Constant:
446   case ISD::ConstantFP: {
447     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
448         N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
449       break;
450 
451     uint64_t Imm;
452     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
453       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
454     else {
455       ConstantSDNode *C = cast<ConstantSDNode>(N);
456       Imm = C->getZExtValue();
457     }
458 
459     SDLoc DL(N);
460     SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
461                                 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
462                                                     MVT::i32));
463     SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
464                                 CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
465     const SDValue Ops[] = {
466       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
467       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
468       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
469     };
470 
471     return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
472                                   N->getValueType(0), Ops);
473   }
474   case ISD::LOAD:
475   case ISD::STORE: {
476     N = glueCopyToM0(N);
477     break;
478   }
479 
480   case AMDGPUISD::BFE_I32:
481   case AMDGPUISD::BFE_U32: {
482     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
483       break;
484 
485     // There is a scalar version available, but unlike the vector version which
486     // has a separate operand for the offset and width, the scalar version packs
487     // the width and offset into a single operand. Try to move to the scalar
488     // version if the offsets are constant, so that we can try to keep extended
489     // loads of kernel arguments in SGPRs.
490 
491     // TODO: Technically we could try to pattern match scalar bitshifts of
492     // dynamic values, but it's probably not useful.
493     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
494     if (!Offset)
495       break;
496 
497     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
498     if (!Width)
499       break;
500 
501     bool Signed = Opc == AMDGPUISD::BFE_I32;
502 
503     uint32_t OffsetVal = Offset->getZExtValue();
504     uint32_t WidthVal = Width->getZExtValue();
505 
506     return getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, SDLoc(N),
507                     N->getOperand(0), OffsetVal, WidthVal);
508   }
509   case AMDGPUISD::DIV_SCALE: {
510     return SelectDIV_SCALE(N);
511   }
512   case ISD::CopyToReg: {
513     const SITargetLowering& Lowering =
514       *static_cast<const SITargetLowering*>(getTargetLowering());
515     Lowering.legalizeTargetIndependentNode(N, *CurDAG);
516     break;
517   }
518   case ISD::ADDRSPACECAST:
519     return SelectAddrSpaceCast(N);
520   case ISD::AND:
521   case ISD::SRL:
522   case ISD::SRA:
523     if (N->getValueType(0) != MVT::i32 ||
524         Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
525       break;
526 
527     return SelectS_BFE(N);
528   case ISD::BRCOND:
529     return SelectBRCOND(N);
530   }
531 
532   return SelectCode(N);
533 }
534 
535 bool AMDGPUDAGToDAGISel::checkType(const Value *Ptr, unsigned AS) {
536   assert(AS != 0 && "Use checkPrivateAddress instead.");
537   if (!Ptr)
538     return false;
539 
540   return Ptr->getType()->getPointerAddressSpace() == AS;
541 }
542 
543 bool AMDGPUDAGToDAGISel::checkPrivateAddress(const MachineMemOperand *Op) {
544   if (Op->getPseudoValue())
545     return true;
546 
547   if (PointerType *PT = dyn_cast<PointerType>(Op->getValue()->getType()))
548     return PT->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS;
549 
550   return false;
551 }
552 
553 bool AMDGPUDAGToDAGISel::isGlobalStore(const StoreSDNode *N) {
554   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS);
555 }
556 
557 bool AMDGPUDAGToDAGISel::isPrivateStore(const StoreSDNode *N) {
558   const Value *MemVal = N->getMemOperand()->getValue();
559   return (!checkType(MemVal, AMDGPUAS::LOCAL_ADDRESS) &&
560           !checkType(MemVal, AMDGPUAS::GLOBAL_ADDRESS) &&
561           !checkType(MemVal, AMDGPUAS::REGION_ADDRESS));
562 }
563 
564 bool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) {
565   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS);
566 }
567 
568 bool AMDGPUDAGToDAGISel::isFlatStore(const StoreSDNode *N) {
569   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::FLAT_ADDRESS);
570 }
571 
572 bool AMDGPUDAGToDAGISel::isRegionStore(const StoreSDNode *N) {
573   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS);
574 }
575 
576 bool AMDGPUDAGToDAGISel::isConstantLoad(const LoadSDNode *N, int CbId) const {
577   const Value *MemVal = N->getMemOperand()->getValue();
578   if (CbId == -1)
579     return checkType(MemVal, AMDGPUAS::CONSTANT_ADDRESS);
580 
581   return checkType(MemVal, AMDGPUAS::CONSTANT_BUFFER_0 + CbId);
582 }
583 
584 bool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) const {
585   if (N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS)
586     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
587         N->getMemoryVT().bitsLT(MVT::i32))
588       return true;
589 
590   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS);
591 }
592 
593 bool AMDGPUDAGToDAGISel::isParamLoad(const LoadSDNode *N) const {
594   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS);
595 }
596 
597 bool AMDGPUDAGToDAGISel::isLocalLoad(const  LoadSDNode *N) const {
598   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS);
599 }
600 
601 bool AMDGPUDAGToDAGISel::isFlatLoad(const  LoadSDNode *N) const {
602   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::FLAT_ADDRESS);
603 }
604 
605 bool AMDGPUDAGToDAGISel::isRegionLoad(const  LoadSDNode *N) const {
606   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS);
607 }
608 
609 bool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) const {
610   MachineMemOperand *MMO = N->getMemOperand();
611   if (checkPrivateAddress(N->getMemOperand())) {
612     if (MMO) {
613       const PseudoSourceValue *PSV = MMO->getPseudoValue();
614       if (PSV && PSV->isConstantPool()) {
615         return true;
616       }
617     }
618   }
619   return false;
620 }
621 
622 bool AMDGPUDAGToDAGISel::isPrivateLoad(const LoadSDNode *N) const {
623   if (checkPrivateAddress(N->getMemOperand())) {
624     // Check to make sure we are not a constant pool load or a constant load
625     // that is marked as a private load
626     if (isCPLoad(N) || isConstantLoad(N, -1)) {
627       return false;
628     }
629   }
630 
631   const Value *MemVal = N->getMemOperand()->getValue();
632   if (!checkType(MemVal, AMDGPUAS::LOCAL_ADDRESS) &&
633       !checkType(MemVal, AMDGPUAS::GLOBAL_ADDRESS) &&
634       !checkType(MemVal, AMDGPUAS::FLAT_ADDRESS) &&
635       !checkType(MemVal, AMDGPUAS::REGION_ADDRESS) &&
636       !checkType(MemVal, AMDGPUAS::CONSTANT_ADDRESS) &&
637       !checkType(MemVal, AMDGPUAS::PARAM_D_ADDRESS) &&
638       !checkType(MemVal, AMDGPUAS::PARAM_I_ADDRESS)) {
639     return true;
640   }
641   return false;
642 }
643 
644 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
645   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
646   return BB->getTerminator()->getMetadata("amdgpu.uniform");
647 }
648 
649 const char *AMDGPUDAGToDAGISel::getPassName() const {
650   return "AMDGPU DAG->DAG Pattern Instruction Selection";
651 }
652 
653 //===----------------------------------------------------------------------===//
654 // Complex Patterns
655 //===----------------------------------------------------------------------===//
656 
657 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
658                                                          SDValue& IntPtr) {
659   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
660     IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
661                                        true);
662     return true;
663   }
664   return false;
665 }
666 
667 bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
668     SDValue& BaseReg, SDValue &Offset) {
669   if (!isa<ConstantSDNode>(Addr)) {
670     BaseReg = Addr;
671     Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
672     return true;
673   }
674   return false;
675 }
676 
677 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
678                                            SDValue &Offset) {
679   ConstantSDNode *IMMOffset;
680 
681   if (Addr.getOpcode() == ISD::ADD
682       && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
683       && isInt<16>(IMMOffset->getZExtValue())) {
684 
685       Base = Addr.getOperand(0);
686       Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
687                                          MVT::i32);
688       return true;
689   // If the pointer address is constant, we can move it to the offset field.
690   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
691              && isInt<16>(IMMOffset->getZExtValue())) {
692     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
693                                   SDLoc(CurDAG->getEntryNode()),
694                                   AMDGPU::ZERO, MVT::i32);
695     Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
696                                        MVT::i32);
697     return true;
698   }
699 
700   // Default case, no offset
701   Base = Addr;
702   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
703   return true;
704 }
705 
706 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
707                                             SDValue &Offset) {
708   ConstantSDNode *C;
709   SDLoc DL(Addr);
710 
711   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
712     Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
713     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
714   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
715             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
716     Base = Addr.getOperand(0);
717     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
718   } else {
719     Base = Addr;
720     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
721   }
722 
723   return true;
724 }
725 
726 SDNode *AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
727   SDLoc DL(N);
728   SDValue LHS = N->getOperand(0);
729   SDValue RHS = N->getOperand(1);
730 
731   bool IsAdd = (N->getOpcode() == ISD::ADD);
732 
733   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
734   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
735 
736   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
737                                        DL, MVT::i32, LHS, Sub0);
738   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
739                                        DL, MVT::i32, LHS, Sub1);
740 
741   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
742                                        DL, MVT::i32, RHS, Sub0);
743   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
744                                        DL, MVT::i32, RHS, Sub1);
745 
746   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
747   SDValue AddLoArgs[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
748 
749 
750   unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
751   unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
752 
753   SDNode *AddLo = CurDAG->getMachineNode( Opc, DL, VTList, AddLoArgs);
754   SDValue Carry(AddLo, 1);
755   SDNode *AddHi
756     = CurDAG->getMachineNode(CarryOpc, DL, MVT::i32,
757                              SDValue(Hi0, 0), SDValue(Hi1, 0), Carry);
758 
759   SDValue Args[5] = {
760     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
761     SDValue(AddLo,0),
762     Sub0,
763     SDValue(AddHi,0),
764     Sub1,
765   };
766   return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, MVT::i64, Args);
767 }
768 
769 // We need to handle this here because tablegen doesn't support matching
770 // instructions with multiple outputs.
771 SDNode *AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
772   SDLoc SL(N);
773   EVT VT = N->getValueType(0);
774 
775   assert(VT == MVT::f32 || VT == MVT::f64);
776 
777   unsigned Opc
778     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
779 
780   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
781   // omod
782   SDValue Ops[8];
783 
784   SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
785   SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]);
786   SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]);
787   return CurDAG->SelectNodeTo(N, Opc, VT, MVT::i1, Ops);
788 }
789 
790 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
791                                          unsigned OffsetBits) const {
792   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
793       (OffsetBits == 8 && !isUInt<8>(Offset)))
794     return false;
795 
796   if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
797       Subtarget->unsafeDSOffsetFoldingEnabled())
798     return true;
799 
800   // On Southern Islands instruction with a negative base value and an offset
801   // don't seem to work.
802   return CurDAG->SignBitIsZero(Base);
803 }
804 
805 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
806                                               SDValue &Offset) const {
807   if (CurDAG->isBaseWithConstantOffset(Addr)) {
808     SDValue N0 = Addr.getOperand(0);
809     SDValue N1 = Addr.getOperand(1);
810     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
811     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
812       // (add n0, c0)
813       Base = N0;
814       Offset = N1;
815       return true;
816     }
817   } else if (Addr.getOpcode() == ISD::SUB) {
818     // sub C, x -> add (sub 0, x), C
819     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
820       int64_t ByteOffset = C->getSExtValue();
821       if (isUInt<16>(ByteOffset)) {
822         SDLoc DL(Addr);
823         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
824 
825         // XXX - This is kind of hacky. Create a dummy sub node so we can check
826         // the known bits in isDSOffsetLegal. We need to emit the selected node
827         // here, so this is thrown away.
828         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
829                                       Zero, Addr.getOperand(1));
830 
831         if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
832           MachineSDNode *MachineSub
833             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
834                                      Zero, Addr.getOperand(1));
835 
836           Base = SDValue(MachineSub, 0);
837           Offset = Addr.getOperand(0);
838           return true;
839         }
840       }
841     }
842   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
843     // If we have a constant address, prefer to put the constant into the
844     // offset. This can save moves to load the constant address since multiple
845     // operations can share the zero base address register, and enables merging
846     // into read2 / write2 instructions.
847 
848     SDLoc DL(Addr);
849 
850     if (isUInt<16>(CAddr->getZExtValue())) {
851       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
852       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
853                                  DL, MVT::i32, Zero);
854       Base = SDValue(MovZero, 0);
855       Offset = Addr;
856       return true;
857     }
858   }
859 
860   // default case
861   Base = Addr;
862   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
863   return true;
864 }
865 
866 // TODO: If offset is too big, put low 16-bit into offset.
867 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
868                                                    SDValue &Offset0,
869                                                    SDValue &Offset1) const {
870   SDLoc DL(Addr);
871 
872   if (CurDAG->isBaseWithConstantOffset(Addr)) {
873     SDValue N0 = Addr.getOperand(0);
874     SDValue N1 = Addr.getOperand(1);
875     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
876     unsigned DWordOffset0 = C1->getZExtValue() / 4;
877     unsigned DWordOffset1 = DWordOffset0 + 1;
878     // (add n0, c0)
879     if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
880       Base = N0;
881       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
882       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
883       return true;
884     }
885   } else if (Addr.getOpcode() == ISD::SUB) {
886     // sub C, x -> add (sub 0, x), C
887     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
888       unsigned DWordOffset0 = C->getZExtValue() / 4;
889       unsigned DWordOffset1 = DWordOffset0 + 1;
890 
891       if (isUInt<8>(DWordOffset0)) {
892         SDLoc DL(Addr);
893         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
894 
895         // XXX - This is kind of hacky. Create a dummy sub node so we can check
896         // the known bits in isDSOffsetLegal. We need to emit the selected node
897         // here, so this is thrown away.
898         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
899                                       Zero, Addr.getOperand(1));
900 
901         if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
902           MachineSDNode *MachineSub
903             = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL, MVT::i32,
904                                      Zero, Addr.getOperand(1));
905 
906           Base = SDValue(MachineSub, 0);
907           Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
908           Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
909           return true;
910         }
911       }
912     }
913   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
914     unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
915     unsigned DWordOffset1 = DWordOffset0 + 1;
916     assert(4 * DWordOffset0 == CAddr->getZExtValue());
917 
918     if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
919       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
920       MachineSDNode *MovZero
921         = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
922                                  DL, MVT::i32, Zero);
923       Base = SDValue(MovZero, 0);
924       Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
925       Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
926       return true;
927     }
928   }
929 
930   // default case
931   Base = Addr;
932   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
933   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
934   return true;
935 }
936 
937 static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm) {
938   return isUInt<12>(Imm->getZExtValue());
939 }
940 
941 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
942                                      SDValue &VAddr, SDValue &SOffset,
943                                      SDValue &Offset, SDValue &Offen,
944                                      SDValue &Idxen, SDValue &Addr64,
945                                      SDValue &GLC, SDValue &SLC,
946                                      SDValue &TFE) const {
947   // Subtarget prefers to use flat instruction
948   if (Subtarget->useFlatForGlobal())
949     return false;
950 
951   SDLoc DL(Addr);
952 
953   GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
954   SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
955   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
956 
957   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
958   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
959   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
960   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
961 
962   if (CurDAG->isBaseWithConstantOffset(Addr)) {
963     SDValue N0 = Addr.getOperand(0);
964     SDValue N1 = Addr.getOperand(1);
965     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
966 
967     if (N0.getOpcode() == ISD::ADD) {
968       // (add (add N2, N3), C1) -> addr64
969       SDValue N2 = N0.getOperand(0);
970       SDValue N3 = N0.getOperand(1);
971       Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
972       Ptr = N2;
973       VAddr = N3;
974     } else {
975 
976       // (add N0, C1) -> offset
977       VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
978       Ptr = N0;
979     }
980 
981     if (isLegalMUBUFImmOffset(C1)) {
982         Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
983         return true;
984     } else if (isUInt<32>(C1->getZExtValue())) {
985       // Illegal offset, store it in soffset.
986       Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
987       SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
988                    CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
989                         0);
990       return true;
991     }
992   }
993 
994   if (Addr.getOpcode() == ISD::ADD) {
995     // (add N0, N1) -> addr64
996     SDValue N0 = Addr.getOperand(0);
997     SDValue N1 = Addr.getOperand(1);
998     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
999     Ptr = N0;
1000     VAddr = N1;
1001     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1002     return true;
1003   }
1004 
1005   // default case -> offset
1006   VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1007   Ptr = Addr;
1008   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1009 
1010   return true;
1011 }
1012 
1013 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1014                                            SDValue &VAddr, SDValue &SOffset,
1015                                            SDValue &Offset, SDValue &GLC,
1016                                            SDValue &SLC, SDValue &TFE) const {
1017   SDValue Ptr, Offen, Idxen, Addr64;
1018 
1019   // addr64 bit was removed for volcanic islands.
1020   if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
1021     return false;
1022 
1023   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1024               GLC, SLC, TFE))
1025     return false;
1026 
1027   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1028   if (C->getSExtValue()) {
1029     SDLoc DL(Addr);
1030 
1031     const SITargetLowering& Lowering =
1032       *static_cast<const SITargetLowering*>(getTargetLowering());
1033 
1034     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1035     return true;
1036   }
1037 
1038   return false;
1039 }
1040 
1041 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1042                                            SDValue &VAddr, SDValue &SOffset,
1043                                            SDValue &Offset,
1044                                            SDValue &SLC) const {
1045   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1046   SDValue GLC, TFE;
1047 
1048   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
1049 }
1050 
1051 bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
1052                                             SDValue &VAddr, SDValue &SOffset,
1053                                             SDValue &ImmOffset) const {
1054 
1055   SDLoc DL(Addr);
1056   MachineFunction &MF = CurDAG->getMachineFunction();
1057   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1058 
1059   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1060   SOffset = CurDAG->getRegister(Info->getScratchWaveOffsetReg(), MVT::i32);
1061 
1062   // (add n0, c1)
1063   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1064     SDValue N0 = Addr.getOperand(0);
1065     SDValue N1 = Addr.getOperand(1);
1066 
1067     // Offsets in vaddr must be positive.
1068     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1069     if (isLegalMUBUFImmOffset(C1) && CurDAG->SignBitIsZero(N0)) {
1070       VAddr = N0;
1071       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1072       return true;
1073     }
1074   }
1075 
1076   // (node)
1077   VAddr = Addr;
1078   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1079   return true;
1080 }
1081 
1082 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1083                                            SDValue &SOffset, SDValue &Offset,
1084                                            SDValue &GLC, SDValue &SLC,
1085                                            SDValue &TFE) const {
1086   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1087   const SIInstrInfo *TII =
1088     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1089 
1090   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1091               GLC, SLC, TFE))
1092     return false;
1093 
1094   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1095       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1096       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1097     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1098                     APInt::getAllOnesValue(32).getZExtValue(); // Size
1099     SDLoc DL(Addr);
1100 
1101     const SITargetLowering& Lowering =
1102       *static_cast<const SITargetLowering*>(getTargetLowering());
1103 
1104     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1105     return true;
1106   }
1107   return false;
1108 }
1109 
1110 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1111                                            SDValue &Soffset, SDValue &Offset,
1112                                            SDValue &GLC) const {
1113   SDValue SLC, TFE;
1114 
1115   return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1116 }
1117 
1118 ///
1119 /// \param EncodedOffset This is the immediate value that will be encoded
1120 ///        directly into the instruction.  On SI/CI the \p EncodedOffset
1121 ///        will be in units of dwords and on VI+ it will be units of bytes.
1122 static bool isLegalSMRDImmOffset(const AMDGPUSubtarget *ST,
1123                                  int64_t EncodedOffset) {
1124   return ST->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
1125      isUInt<8>(EncodedOffset) : isUInt<20>(EncodedOffset);
1126 }
1127 
1128 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1129                                           SDValue &Offset, bool &Imm) const {
1130 
1131   // FIXME: Handle non-constant offsets.
1132   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1133   if (!C)
1134     return false;
1135 
1136   SDLoc SL(ByteOffsetNode);
1137   AMDGPUSubtarget::Generation Gen = Subtarget->getGeneration();
1138   int64_t ByteOffset = C->getSExtValue();
1139   int64_t EncodedOffset = Gen < AMDGPUSubtarget::VOLCANIC_ISLANDS ?
1140       ByteOffset >> 2 : ByteOffset;
1141 
1142   if (isLegalSMRDImmOffset(Subtarget, EncodedOffset)) {
1143     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1144     Imm = true;
1145     return true;
1146   }
1147 
1148   if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
1149     return false;
1150 
1151   if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
1152     // 32-bit Immediates are supported on Sea Islands.
1153     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1154   } else {
1155     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1156     Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
1157                                             C32Bit), 0);
1158   }
1159   Imm = false;
1160   return true;
1161 }
1162 
1163 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1164                                      SDValue &Offset, bool &Imm) const {
1165 
1166   SDLoc SL(Addr);
1167   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1168     SDValue N0 = Addr.getOperand(0);
1169     SDValue N1 = Addr.getOperand(1);
1170 
1171     if (SelectSMRDOffset(N1, Offset, Imm)) {
1172       SBase = N0;
1173       return true;
1174     }
1175   }
1176   SBase = Addr;
1177   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
1178   Imm = true;
1179   return true;
1180 }
1181 
1182 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1183                                        SDValue &Offset) const {
1184   bool Imm;
1185   return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
1186 }
1187 
1188 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1189                                          SDValue &Offset) const {
1190 
1191   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1192     return false;
1193 
1194   bool Imm;
1195   if (!SelectSMRD(Addr, SBase, Offset, Imm))
1196     return false;
1197 
1198   return !Imm && isa<ConstantSDNode>(Offset);
1199 }
1200 
1201 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1202                                         SDValue &Offset) const {
1203   bool Imm;
1204   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
1205          !isa<ConstantSDNode>(Offset);
1206 }
1207 
1208 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
1209                                              SDValue &Offset) const {
1210   bool Imm;
1211   return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
1212 }
1213 
1214 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
1215                                                SDValue &Offset) const {
1216   if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1217     return false;
1218 
1219   bool Imm;
1220   if (!SelectSMRDOffset(Addr, Offset, Imm))
1221     return false;
1222 
1223   return !Imm && isa<ConstantSDNode>(Offset);
1224 }
1225 
1226 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(SDValue Addr,
1227                                               SDValue &Offset) const {
1228   bool Imm;
1229   return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
1230          !isa<ConstantSDNode>(Offset);
1231 }
1232 
1233 // FIXME: This is incorrect and only enough to be able to compile.
1234 SDNode *AMDGPUDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
1235   AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(N);
1236   SDLoc DL(N);
1237 
1238   const MachineFunction &MF = CurDAG->getMachineFunction();
1239   DiagnosticInfoUnsupported NotImplemented(
1240       *MF.getFunction(), "addrspacecast not implemented", DL.getDebugLoc());
1241   CurDAG->getContext()->diagnose(NotImplemented);
1242 
1243   assert(Subtarget->hasFlatAddressSpace() &&
1244          "addrspacecast only supported with flat address space!");
1245 
1246   assert((ASC->getSrcAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
1247           ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS) &&
1248          "Can only cast to / from flat address space!");
1249 
1250   // The flat instructions read the address as the index of the VGPR holding the
1251   // address, so casting should just be reinterpreting the base VGPR, so just
1252   // insert trunc / bitcast / zext.
1253 
1254   SDValue Src = ASC->getOperand(0);
1255   EVT DestVT = ASC->getValueType(0);
1256   EVT SrcVT = Src.getValueType();
1257 
1258   unsigned SrcSize = SrcVT.getSizeInBits();
1259   unsigned DestSize = DestVT.getSizeInBits();
1260 
1261   if (SrcSize > DestSize) {
1262     assert(SrcSize == 64 && DestSize == 32);
1263     return CurDAG->getMachineNode(
1264       TargetOpcode::EXTRACT_SUBREG,
1265       DL,
1266       DestVT,
1267       Src,
1268       CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32));
1269   }
1270 
1271   if (DestSize > SrcSize) {
1272     assert(SrcSize == 32 && DestSize == 64);
1273 
1274     // FIXME: This is probably wrong, we should never be defining
1275     // a register class with both VGPRs and SGPRs
1276     SDValue RC = CurDAG->getTargetConstant(AMDGPU::VS_64RegClassID, DL,
1277                                            MVT::i32);
1278 
1279     const SDValue Ops[] = {
1280       RC,
1281       Src,
1282       CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
1283       SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
1284                                      CurDAG->getConstant(0, DL, MVT::i32)), 0),
1285       CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
1286     };
1287 
1288     return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
1289                                   DL, N->getValueType(0), Ops);
1290   }
1291 
1292   assert(SrcSize == 64 && DestSize == 64);
1293   return CurDAG->getNode(ISD::BITCAST, DL, DestVT, Src).getNode();
1294 }
1295 
1296 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, SDLoc DL, SDValue Val,
1297                                      uint32_t Offset, uint32_t Width) {
1298   // Transformation function, pack the offset and width of a BFE into
1299   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1300   // source, bits [5:0] contain the offset and bits [22:16] the width.
1301   uint32_t PackedVal = Offset | (Width << 16);
1302   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
1303 
1304   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
1305 }
1306 
1307 SDNode *AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
1308   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
1309   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
1310   // Predicate: 0 < b <= c < 32
1311 
1312   const SDValue &Shl = N->getOperand(0);
1313   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
1314   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
1315 
1316   if (B && C) {
1317     uint32_t BVal = B->getZExtValue();
1318     uint32_t CVal = C->getZExtValue();
1319 
1320     if (0 < BVal && BVal <= CVal && CVal < 32) {
1321       bool Signed = N->getOpcode() == ISD::SRA;
1322       unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1323 
1324       return getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0),
1325                       CVal - BVal, 32 - CVal);
1326     }
1327   }
1328   return SelectCode(N);
1329 }
1330 
1331 SDNode *AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
1332   switch (N->getOpcode()) {
1333   case ISD::AND:
1334     if (N->getOperand(0).getOpcode() == ISD::SRL) {
1335       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
1336       // Predicate: isMask(mask)
1337       const SDValue &Srl = N->getOperand(0);
1338       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
1339       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
1340 
1341       if (Shift && Mask) {
1342         uint32_t ShiftVal = Shift->getZExtValue();
1343         uint32_t MaskVal = Mask->getZExtValue();
1344 
1345         if (isMask_32(MaskVal)) {
1346           uint32_t WidthVal = countPopulation(MaskVal);
1347 
1348           return getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), Srl.getOperand(0),
1349                           ShiftVal, WidthVal);
1350         }
1351       }
1352     }
1353     break;
1354   case ISD::SRL:
1355     if (N->getOperand(0).getOpcode() == ISD::AND) {
1356       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
1357       // Predicate: isMask(mask >> b)
1358       const SDValue &And = N->getOperand(0);
1359       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
1360       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
1361 
1362       if (Shift && Mask) {
1363         uint32_t ShiftVal = Shift->getZExtValue();
1364         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1365 
1366         if (isMask_32(MaskVal)) {
1367           uint32_t WidthVal = countPopulation(MaskVal);
1368 
1369           return getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), And.getOperand(0),
1370                           ShiftVal, WidthVal);
1371         }
1372       }
1373     } else if (N->getOperand(0).getOpcode() == ISD::SHL)
1374       return SelectS_BFEFromShifts(N);
1375     break;
1376   case ISD::SRA:
1377     if (N->getOperand(0).getOpcode() == ISD::SHL)
1378       return SelectS_BFEFromShifts(N);
1379     break;
1380   }
1381 
1382   return SelectCode(N);
1383 }
1384 
1385 SDNode *AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
1386   SDValue Cond = N->getOperand(1);
1387 
1388   if (isCBranchSCC(N)) {
1389     // This brcond will use S_CBRANCH_SCC*, so let tablegen handle it.
1390     return SelectCode(N);
1391   }
1392 
1393   // The result of VOPC instructions is or'd against ~EXEC before it is
1394   // written to vcc or another SGPR.  This means that the value '1' is always
1395   // written to the corresponding bit for results that are masked.  In order
1396   // to correctly check against vccz, we need to and VCC with the EXEC
1397   // register in order to clear the value from the masked bits.
1398 
1399   SDLoc SL(N);
1400 
1401   SDNode *MaskedCond =
1402         CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1,
1403                                CurDAG->getRegister(AMDGPU::EXEC, MVT::i1),
1404                                Cond);
1405   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC,
1406                                      SDValue(MaskedCond, 0),
1407                                      SDValue()); // Passing SDValue() adds a
1408                                                  // glue output.
1409   return CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ, MVT::Other,
1410                               N->getOperand(2), // Basic Block
1411                               VCC.getValue(0),  // Chain
1412                               VCC.getValue(1)); // Glue
1413 }
1414 
1415 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
1416                                         SDValue &SrcMods) const {
1417 
1418   unsigned Mods = 0;
1419 
1420   Src = In;
1421 
1422   if (Src.getOpcode() == ISD::FNEG) {
1423     Mods |= SISrcMods::NEG;
1424     Src = Src.getOperand(0);
1425   }
1426 
1427   if (Src.getOpcode() == ISD::FABS) {
1428     Mods |= SISrcMods::ABS;
1429     Src = Src.getOperand(0);
1430   }
1431 
1432   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
1433 
1434   return true;
1435 }
1436 
1437 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src,
1438                                          SDValue &SrcMods) const {
1439   bool Res = SelectVOP3Mods(In, Src, SrcMods);
1440   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
1441 }
1442 
1443 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
1444                                          SDValue &SrcMods, SDValue &Clamp,
1445                                          SDValue &Omod) const {
1446   SDLoc DL(In);
1447   // FIXME: Handle Clamp and Omod
1448   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i32);
1449   Omod = CurDAG->getTargetConstant(0, DL, MVT::i32);
1450 
1451   return SelectVOP3Mods(In, Src, SrcMods);
1452 }
1453 
1454 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(SDValue In, SDValue &Src,
1455                                            SDValue &SrcMods, SDValue &Clamp,
1456                                            SDValue &Omod) const {
1457   bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
1458 
1459   return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
1460                 cast<ConstantSDNode>(Clamp)->isNullValue() &&
1461                 cast<ConstantSDNode>(Omod)->isNullValue();
1462 }
1463 
1464 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(SDValue In, SDValue &Src,
1465                                               SDValue &SrcMods,
1466                                               SDValue &Omod) const {
1467   // FIXME: Handle Omod
1468   Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1469 
1470   return SelectVOP3Mods(In, Src, SrcMods);
1471 }
1472 
1473 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
1474                                                    SDValue &SrcMods,
1475                                                    SDValue &Clamp,
1476                                                    SDValue &Omod) const {
1477   Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1478   return SelectVOP3Mods(In, Src, SrcMods);
1479 }
1480 
1481 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
1482   bool Modified = false;
1483 
1484   // XXX - Other targets seem to be able to do this without a worklist.
1485   SmallVector<LoadSDNode *, 8> LoadsToReplace;
1486   SmallVector<StoreSDNode *, 8> StoresToReplace;
1487 
1488   for (SDNode &Node : CurDAG->allnodes()) {
1489     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(&Node)) {
1490       EVT VT = LD->getValueType(0);
1491       if (VT != MVT::i64 || LD->getExtensionType() != ISD::NON_EXTLOAD)
1492         continue;
1493 
1494       // To simplify the TableGen patters, we replace all i64 loads with v2i32
1495       // loads.  Alternatively, we could promote i64 loads to v2i32 during DAG
1496       // legalization, however, so places (ExpandUnalignedLoad) in the DAG
1497       // legalizer assume that if i64 is legal, so doing this promotion early
1498       // can cause problems.
1499       LoadsToReplace.push_back(LD);
1500     } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(&Node)) {
1501       // Handle i64 stores here for the same reason mentioned above for loads.
1502       SDValue Value = ST->getValue();
1503       if (Value.getValueType() != MVT::i64 || ST->isTruncatingStore())
1504         continue;
1505       StoresToReplace.push_back(ST);
1506     }
1507   }
1508 
1509   for (LoadSDNode *LD : LoadsToReplace) {
1510     SDLoc SL(LD);
1511 
1512     SDValue NewLoad = CurDAG->getLoad(MVT::v2i32, SL, LD->getChain(),
1513                                       LD->getBasePtr(), LD->getMemOperand());
1514     SDValue BitCast = CurDAG->getNode(ISD::BITCAST, SL,
1515                                       MVT::i64, NewLoad);
1516     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
1517     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LD, 0), BitCast);
1518     Modified = true;
1519   }
1520 
1521   for (StoreSDNode *ST : StoresToReplace) {
1522     SDValue NewValue = CurDAG->getNode(ISD::BITCAST, SDLoc(ST),
1523                                        MVT::v2i32, ST->getValue());
1524     const SDValue StoreOps[] = {
1525       ST->getChain(),
1526       NewValue,
1527       ST->getBasePtr(),
1528       ST->getOffset()
1529     };
1530 
1531     CurDAG->UpdateNodeOperands(ST, StoreOps);
1532     Modified = true;
1533   }
1534 
1535   // XXX - Is this necessary?
1536   if (Modified)
1537     CurDAG->RemoveDeadNodes();
1538 }
1539 
1540 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
1541   const AMDGPUTargetLowering& Lowering =
1542     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
1543   bool IsModified = false;
1544   do {
1545     IsModified = false;
1546     // Go over all selected nodes and try to fold them a bit more
1547     for (SDNode &Node : CurDAG->allnodes()) {
1548       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
1549       if (!MachineNode)
1550         continue;
1551 
1552       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
1553       if (ResNode != &Node) {
1554         ReplaceUses(&Node, ResNode);
1555         IsModified = true;
1556       }
1557     }
1558     CurDAG->RemoveDeadNodes();
1559   } while (IsModified);
1560 }
1561