1 //===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16 
17 #include "GCNSubtarget.h"
18 #include "SIMachineFunctionInfo.h"
19 #include "llvm/CodeGen/SelectionDAGISel.h"
20 #include "llvm/Target/TargetMachine.h"
21 
22 using namespace llvm;
23 
24 namespace {
25 
isNullConstantOrUndef(SDValue V)26 static inline bool isNullConstantOrUndef(SDValue V) {
27   if (V.isUndef())
28     return true;
29 
30   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
31   return Const != nullptr && Const->isZero();
32 }
33 
getConstantValue(SDValue N,uint32_t & Out)34 static inline bool getConstantValue(SDValue N, uint32_t &Out) {
35   // This is only used for packed vectors, where using 0 for undef should
36   // always be good.
37   if (N.isUndef()) {
38     Out = 0;
39     return true;
40   }
41 
42   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
43     Out = C->getAPIntValue().getSExtValue();
44     return true;
45   }
46 
47   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
48     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
49     return true;
50   }
51 
52   return false;
53 }
54 
55 // TODO: Handle undef as zero
56 static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
57                                         bool Negate = false) {
58   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
59   uint32_t LHSVal, RHSVal;
60   if (getConstantValue(N->getOperand(0), LHSVal) &&
61       getConstantValue(N->getOperand(1), RHSVal)) {
62     SDLoc SL(N);
63     uint32_t K = Negate ? (-LHSVal & 0xffff) | (-RHSVal << 16)
64                         : (LHSVal & 0xffff) | (RHSVal << 16);
65     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
66                               DAG.getTargetConstant(K, SL, MVT::i32));
67   }
68 
69   return nullptr;
70 }
71 
packNegConstantV2I16(const SDNode * N,SelectionDAG & DAG)72 static inline SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
73   return packConstantV2I16(N, DAG, true);
74 }
75 } // namespace
76 
77 /// AMDGPU specific code to select AMDGPU machine instructions for
78 /// SelectionDAG operations.
79 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
80   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
81   // make the right decision when generating code for different targets.
82   const GCNSubtarget *Subtarget;
83 
84   // Default FP mode for the current function.
85   AMDGPU::SIModeRegisterDefaults Mode;
86 
87   bool EnableLateStructurizeCFG;
88 
89   // Instructions that will be lowered with a final instruction that zeros the
90   // high result bits.
91   bool fp16SrcZerosHighBits(unsigned Opc) const;
92 
93 public:
94   explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
95                               CodeGenOpt::Level OptLevel = CodeGenOpt::Default);
96   ~AMDGPUDAGToDAGISel() override = default;
97 
98   void getAnalysisUsage(AnalysisUsage &AU) const override;
99 
100   bool matchLoadD16FromBuildVector(SDNode *N) const;
101 
102   bool runOnMachineFunction(MachineFunction &MF) override;
103   void PreprocessISelDAG() override;
104   void Select(SDNode *N) override;
105   StringRef getPassName() const override;
106   void PostprocessISelDAG() override;
107 
108 protected:
109   void SelectBuildVector(SDNode *N, unsigned RegClassID);
110 
111 private:
112   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
113   bool isNoNanSrc(SDValue N) const;
114   bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
isNegInlineImmediate(const SDNode * N)115   bool isNegInlineImmediate(const SDNode *N) const {
116     return isInlineImmediate(N, true);
117   }
118 
isInlineImmediate16(int64_t Imm)119   bool isInlineImmediate16(int64_t Imm) const {
120     return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());
121   }
122 
isInlineImmediate32(int64_t Imm)123   bool isInlineImmediate32(int64_t Imm) const {
124     return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());
125   }
126 
isInlineImmediate64(int64_t Imm)127   bool isInlineImmediate64(int64_t Imm) const {
128     return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());
129   }
130 
isInlineImmediate(const APFloat & Imm)131   bool isInlineImmediate(const APFloat &Imm) const {
132     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
133   }
134 
135   bool isVGPRImm(const SDNode *N) const;
136   bool isUniformLoad(const SDNode *N) const;
137   bool isUniformBr(const SDNode *N) const;
138 
139   // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
140   // `ShAmtBits` bits is unneeded.
141   bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
142 
143   bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
144                                   SDValue &RHS) const;
145 
146   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
147 
148   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
149   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
150   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
151 
152   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
153   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
154   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
155   bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
156   bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
157                         unsigned Size) const;
158   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
159   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
160                                  SDValue &Offset1) const;
161   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
162                                   SDValue &Offset1) const;
163   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
164                           SDValue &Offset1, unsigned Size) const;
165   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
166                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
167                    SDValue &Idxen, SDValue &Addr64) const;
168   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
169                          SDValue &SOffset, SDValue &Offset) const;
170   bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
171                                SDValue &VAddr, SDValue &SOffset,
172                                SDValue &ImmOffset) const;
173   bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
174                                 SDValue &Soffset, SDValue &Offset) const;
175 
176   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
177                          SDValue &Offset) const;
178 
179   bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
180                             SDValue &Offset, uint64_t FlatVariant) const;
181   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
182                         SDValue &Offset) const;
183   bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
184                           SDValue &Offset) const;
185   bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
186                            SDValue &Offset) const;
187   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
188                          SDValue &VOffset, SDValue &Offset) const;
189   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
190                           SDValue &Offset) const;
191   bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
192                                      uint64_t ImmOffset) const;
193   bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
194                            SDValue &SAddr, SDValue &Offset) const;
195 
196   bool SelectSMRDOffset(SDValue Base, SDValue ByteOffsetNode, SDValue *SOffset,
197                         SDValue *Offset, bool Imm32Only = false) const;
198   SDValue Expand32BitAddress(SDValue Addr) const;
199   bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset,
200                             SDValue *Offset, bool Imm32Only = false) const;
201   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset,
202                   SDValue *Offset, bool Imm32Only = false) const;
203   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
204   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
205   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const;
206   bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset,
207                          SDValue &Offset) const;
208   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
209   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
210   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
211 
212   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
213   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
214                           bool AllowAbs = true) const;
215   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
216   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
217   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
218   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
219                        SDValue &Clamp, SDValue &Omod) const;
220   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
221                         SDValue &Clamp, SDValue &Omod) const;
222   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
223                          SDValue &Clamp, SDValue &Omod) const;
224 
225   bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
226                              bool OpSel) const;
227   bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
228   bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
229 
230   bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
231                        SDValue &Omod) const;
232 
233   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
234                        bool IsDOT = false) const;
235   bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
236 
237   bool SelectDotIUVOP3PMods(SDValue In, SDValue &Src) const;
238   bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
239 
240   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
241 
242   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
243   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
244                                  unsigned &Mods) const;
245   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
246 
247   SDValue getHi16Elt(SDValue In) const;
248 
249   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
250 
251   void SelectADD_SUB_I64(SDNode *N);
252   void SelectAddcSubb(SDNode *N);
253   void SelectUADDO_USUBO(SDNode *N);
254   void SelectDIV_SCALE(SDNode *N);
255   void SelectMAD_64_32(SDNode *N);
256   void SelectMUL_LOHI(SDNode *N);
257   void SelectFMA_W_CHAIN(SDNode *N);
258   void SelectFMUL_W_CHAIN(SDNode *N);
259   SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
260                    uint32_t Width);
261   void SelectS_BFEFromShifts(SDNode *N);
262   void SelectS_BFE(SDNode *N);
263   bool isCBranchSCC(const SDNode *N) const;
264   void SelectBRCOND(SDNode *N);
265   void SelectFMAD_FMA(SDNode *N);
266   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
267   void SelectDS_GWS(SDNode *N, unsigned IntrID);
268   void SelectInterpP1F16(SDNode *N);
269   void SelectINTRINSIC_W_CHAIN(SDNode *N);
270   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
271   void SelectINTRINSIC_VOID(SDNode *N);
272 
273 protected:
274   // Include the pieces autogenerated from the target description.
275 #include "AMDGPUGenDAGISel.inc"
276 };
277 
278 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
279