1 //===-- AMDGPUISelLowering.h - AMDGPU Lowering Interface --------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// Interface definition of the TargetLowering class that is common
12 /// to all AMD GPUs.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELLOWERING_H
17 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELLOWERING_H
18 
19 #include "AMDGPU.h"
20 #include "llvm/CodeGen/CallingConvLower.h"
21 #include "llvm/CodeGen/TargetLowering.h"
22 
23 namespace llvm {
24 
25 class AMDGPUMachineFunction;
26 class AMDGPUSubtarget;
27 struct ArgDescriptor;
28 
29 class AMDGPUTargetLowering : public TargetLowering {
30 private:
31   const AMDGPUSubtarget *Subtarget;
32 
33   /// \returns AMDGPUISD::FFBH_U32 node if the incoming \p Op may have been
34   /// legalized from a smaller type VT. Need to match pre-legalized type because
35   /// the generic legalization inserts the add/sub between the select and
36   /// compare.
37   SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const;
38 
39 public:
40   static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG);
41   static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG);
42 
43 protected:
44   SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
45   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
46   /// Split a vector store into multiple scalar stores.
47   /// \returns The resulting chain.
48 
49   SDValue LowerFREM(SDValue Op, SelectionDAG &DAG) const;
50   SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const;
51   SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const;
52   SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const;
53   SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const;
54 
55   SDValue LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const;
56   SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
57   SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
58   SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const;
59   SDValue LowerFLOG(SDValue Op, SelectionDAG &DAG,
60                     double Log2BaseInverted) const;
61   SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const;
62 
63   SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const;
64 
65   SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const;
66   SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const;
67   SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
68   SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
69 
70   SDValue LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, bool Signed) const;
71   SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const;
72   SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
73   SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
74 
75   SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
76 
77 protected:
78   bool shouldCombineMemoryType(EVT VT) const;
79   SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const;
80   SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const;
81   SDValue performAssertSZExtCombine(SDNode *N, DAGCombinerInfo &DCI) const;
82 
83   SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL,
84                                        unsigned Opc, SDValue LHS,
85                                        uint32_t ValLo, uint32_t ValHi) const;
86   SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
87   SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const;
88   SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
89   SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const;
90   SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
91   SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
92   SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
93   SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const;
94   SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
95                              SDValue RHS, DAGCombinerInfo &DCI) const;
96   SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
97 
98   bool isConstantCostlierToNegate(SDValue N) const;
99   SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
100   SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
101   SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
102 
103   static EVT getEquivalentMemType(LLVMContext &Context, EVT VT);
104 
105   virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
106                                      SelectionDAG &DAG) const;
107 
108   /// Return 64-bit value Op as two 32-bit integers.
109   std::pair<SDValue, SDValue> split64BitValue(SDValue Op,
110                                               SelectionDAG &DAG) const;
111   SDValue getLoHalf64(SDValue Op, SelectionDAG &DAG) const;
112   SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const;
113 
114   /// Split a vector load into 2 loads of half the vector.
115   SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const;
116 
117   /// Split a vector store into 2 stores of half the vector.
118   SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const;
119 
120   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
121   SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
122   SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
123   SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const;
124   void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG,
125                                     SmallVectorImpl<SDValue> &Results) const;
126 
127   void analyzeFormalArgumentsCompute(
128     CCState &State,
129     const SmallVectorImpl<ISD::InputArg> &Ins) const;
130 
131 public:
132   AMDGPUTargetLowering(const TargetMachine &TM, const AMDGPUSubtarget &STI);
133 
134   bool mayIgnoreSignedZero(SDValue Op) const {
135     if (getTargetMachine().Options.NoSignedZerosFPMath)
136       return true;
137 
138     const auto Flags = Op.getNode()->getFlags();
139     if (Flags.isDefined())
140       return Flags.hasNoSignedZeros();
141 
142     return false;
143   }
144 
145   static inline SDValue stripBitcast(SDValue Val) {
146     return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
147   }
148 
149   static bool allUsesHaveSourceMods(const SDNode *N,
150                                     unsigned CostThreshold = 4);
151   bool isFAbsFree(EVT VT) const override;
152   bool isFNegFree(EVT VT) const override;
153   bool isTruncateFree(EVT Src, EVT Dest) const override;
154   bool isTruncateFree(Type *Src, Type *Dest) const override;
155 
156   bool isZExtFree(Type *Src, Type *Dest) const override;
157   bool isZExtFree(EVT Src, EVT Dest) const override;
158   bool isZExtFree(SDValue Val, EVT VT2) const override;
159 
160   bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
161 
162   MVT getVectorIdxTy(const DataLayout &) const override;
163   bool isSelectSupported(SelectSupportKind) const override;
164 
165   bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
166   bool ShouldShrinkFPConstant(EVT VT) const override;
167   bool shouldReduceLoadWidth(SDNode *Load,
168                              ISD::LoadExtType ExtType,
169                              EVT ExtVT) const override;
170 
171   bool isLoadBitCastBeneficial(EVT, EVT) const final;
172 
173   bool storeOfVectorConstantIsCheap(EVT MemVT,
174                                     unsigned NumElem,
175                                     unsigned AS) const override;
176   bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override;
177   bool isCheapToSpeculateCttz() const override;
178   bool isCheapToSpeculateCtlz() const override;
179 
180   bool isSDNodeAlwaysUniform(const SDNode *N) const override;
181   static CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg);
182   static CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg);
183 
184   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
185                       const SmallVectorImpl<ISD::OutputArg> &Outs,
186                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
187                       SelectionDAG &DAG) const override;
188 
189   SDValue addTokenForArgument(SDValue Chain,
190                               SelectionDAG &DAG,
191                               MachineFrameInfo &MFI,
192                               int ClobberedFI) const;
193 
194   SDValue lowerUnhandledCall(CallLoweringInfo &CLI,
195                              SmallVectorImpl<SDValue> &InVals,
196                              StringRef Reason) const;
197   SDValue LowerCall(CallLoweringInfo &CLI,
198                     SmallVectorImpl<SDValue> &InVals) const override;
199 
200   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op,
201                                   SelectionDAG &DAG) const;
202 
203   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
204   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
205   void ReplaceNodeResults(SDNode * N,
206                           SmallVectorImpl<SDValue> &Results,
207                           SelectionDAG &DAG) const override;
208 
209   SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS,
210                                SDValue RHS, SDValue True, SDValue False,
211                                SDValue CC, DAGCombinerInfo &DCI) const;
212 
213   const char* getTargetNodeName(unsigned Opcode) const override;
214 
215   // FIXME: Turn off MergeConsecutiveStores() before Instruction Selection
216   // for AMDGPU.
217   // A commit ( git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@319036
218   // 91177308-0d34-0410-b5e6-96231b3b80d8 ) turned on
219   // MergeConsecutiveStores() before Instruction Selection for all targets.
220   // Enough AMDGPU compiles go into an infinite loop ( MergeConsecutiveStores()
221   // merges two stores; LegalizeStoreOps() un-merges; MergeConsecutiveStores()
222   // re-merges, etc. ) to warrant turning it off for now.
223   bool mergeStoresAfterLegalization() const override { return false; }
224 
225   bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override {
226     return true;
227   }
228   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
229                            int &RefinementSteps, bool &UseOneConstNR,
230                            bool Reciprocal) const override;
231   SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
232                            int &RefinementSteps) const override;
233 
234   virtual SDNode *PostISelFolding(MachineSDNode *N,
235                                   SelectionDAG &DAG) const = 0;
236 
237   /// Determine which of the bits specified in \p Mask are known to be
238   /// either zero or one and return them in the \p KnownZero and \p KnownOne
239   /// bitsets.
240   void computeKnownBitsForTargetNode(const SDValue Op,
241                                      KnownBits &Known,
242                                      const APInt &DemandedElts,
243                                      const SelectionDAG &DAG,
244                                      unsigned Depth = 0) const override;
245 
246   unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts,
247                                            const SelectionDAG &DAG,
248                                            unsigned Depth = 0) const override;
249 
250   bool isKnownNeverNaNForTargetNode(SDValue Op,
251                                     const SelectionDAG &DAG,
252                                     bool SNaN = false,
253                                     unsigned Depth = 0) const override;
254 
255   /// Helper function that adds Reg to the LiveIn list of the DAG's
256   /// MachineFunction.
257   ///
258   /// \returns a RegisterSDNode representing Reg if \p RawReg is true, otherwise
259   /// a copy from the register.
260   SDValue CreateLiveInRegister(SelectionDAG &DAG,
261                                const TargetRegisterClass *RC,
262                                unsigned Reg, EVT VT,
263                                const SDLoc &SL,
264                                bool RawReg = false) const;
265   SDValue CreateLiveInRegister(SelectionDAG &DAG,
266                                const TargetRegisterClass *RC,
267                                unsigned Reg, EVT VT) const {
268     return CreateLiveInRegister(DAG, RC, Reg, VT, SDLoc(DAG.getEntryNode()));
269   }
270 
271   // Returns the raw live in register rather than a copy from it.
272   SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG,
273                                   const TargetRegisterClass *RC,
274                                   unsigned Reg, EVT VT) const {
275     return CreateLiveInRegister(DAG, RC, Reg, VT, SDLoc(DAG.getEntryNode()), true);
276   }
277 
278   /// Similar to CreateLiveInRegister, except value maybe loaded from a stack
279   /// slot rather than passed in a register.
280   SDValue loadStackInputValue(SelectionDAG &DAG,
281                               EVT VT,
282                               const SDLoc &SL,
283                               int64_t Offset) const;
284 
285   SDValue storeStackInputValue(SelectionDAG &DAG,
286                                const SDLoc &SL,
287                                SDValue Chain,
288                                SDValue ArgVal,
289                                int64_t Offset) const;
290 
291   SDValue loadInputValue(SelectionDAG &DAG,
292                          const TargetRegisterClass *RC,
293                          EVT VT, const SDLoc &SL,
294                          const ArgDescriptor &Arg) const;
295 
296   enum ImplicitParameter {
297     FIRST_IMPLICIT,
298     GRID_DIM = FIRST_IMPLICIT,
299     GRID_OFFSET,
300   };
301 
302   /// Helper function that returns the byte offset of the given
303   /// type of implicit parameter.
304   uint32_t getImplicitParameterOffset(const MachineFunction &MF,
305                                       const ImplicitParameter Param) const;
306 
307   MVT getFenceOperandTy(const DataLayout &DL) const override {
308     return MVT::i32;
309   }
310 
311   AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
312 };
313 
314 namespace AMDGPUISD {
315 
316 enum NodeType : unsigned {
317   // AMDIL ISD Opcodes
318   FIRST_NUMBER = ISD::BUILTIN_OP_END,
319   UMUL,        // 32bit unsigned multiplication
320   BRANCH_COND,
321   // End AMDIL ISD Opcodes
322 
323   // Function call.
324   CALL,
325   TC_RETURN,
326   TRAP,
327 
328   // Masked control flow nodes.
329   IF,
330   ELSE,
331   LOOP,
332 
333   // A uniform kernel return that terminates the wavefront.
334   ENDPGM,
335 
336   // Return to a shader part's epilog code.
337   RETURN_TO_EPILOG,
338 
339   // Return with values from a non-entry function.
340   RET_FLAG,
341 
342   DWORDADDR,
343   FRACT,
344 
345   /// CLAMP value between 0.0 and 1.0. NaN clamped to 0, following clamp output
346   /// modifier behavior with dx10_enable.
347   CLAMP,
348 
349   // This is SETCC with the full mask result which is used for a compare with a
350   // result bit per item in the wavefront.
351   SETCC,
352   SETREG,
353   // FP ops with input and output chain.
354   FMA_W_CHAIN,
355   FMUL_W_CHAIN,
356 
357   // SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi.
358   // Denormals handled on some parts.
359   COS_HW,
360   SIN_HW,
361   FMAX_LEGACY,
362   FMIN_LEGACY,
363 
364   FMAX3,
365   SMAX3,
366   UMAX3,
367   FMIN3,
368   SMIN3,
369   UMIN3,
370   FMED3,
371   SMED3,
372   UMED3,
373   FDOT2,
374   URECIP,
375   DIV_SCALE,
376   DIV_FMAS,
377   DIV_FIXUP,
378   // For emitting ISD::FMAD when f32 denormals are enabled because mac/mad is
379   // treated as an illegal operation.
380   FMAD_FTZ,
381   TRIG_PREOP, // 1 ULP max error for f64
382 
383   // RCP, RSQ - For f32, 1 ULP max error, no denormal handling.
384   //            For f64, max error 2^29 ULP, handles denormals.
385   RCP,
386   RSQ,
387   RCP_LEGACY,
388   RSQ_LEGACY,
389   RCP_IFLAG,
390   FMUL_LEGACY,
391   RSQ_CLAMP,
392   LDEXP,
393   FP_CLASS,
394   DOT4,
395   CARRY,
396   BORROW,
397   BFE_U32, // Extract range of bits with zero extension to 32-bits.
398   BFE_I32, // Extract range of bits with sign extension to 32-bits.
399   BFI, // (src0 & src1) | (~src0 & src2)
400   BFM, // Insert a range of bits into a 32-bit word.
401   FFBH_U32, // ctlz with -1 if input is zero.
402   FFBH_I32,
403   FFBL_B32, // cttz with -1 if input is zero.
404   MUL_U24,
405   MUL_I24,
406   MULHI_U24,
407   MULHI_I24,
408   MAD_U24,
409   MAD_I24,
410   MAD_U64_U32,
411   MAD_I64_I32,
412   MUL_LOHI_I24,
413   MUL_LOHI_U24,
414   PERM,
415   TEXTURE_FETCH,
416   EXPORT, // exp on SI+
417   EXPORT_DONE, // exp on SI+ with done bit set
418   R600_EXPORT,
419   CONST_ADDRESS,
420   REGISTER_LOAD,
421   REGISTER_STORE,
422   SAMPLE,
423   SAMPLEB,
424   SAMPLED,
425   SAMPLEL,
426 
427   // These cvt_f32_ubyte* nodes need to remain consecutive and in order.
428   CVT_F32_UBYTE0,
429   CVT_F32_UBYTE1,
430   CVT_F32_UBYTE2,
431   CVT_F32_UBYTE3,
432 
433   // Convert two float 32 numbers into a single register holding two packed f16
434   // with round to zero.
435   CVT_PKRTZ_F16_F32,
436   CVT_PKNORM_I16_F32,
437   CVT_PKNORM_U16_F32,
438   CVT_PK_I16_I32,
439   CVT_PK_U16_U32,
440 
441   // Same as the standard node, except the high bits of the resulting integer
442   // are known 0.
443   FP_TO_FP16,
444 
445   // Wrapper around fp16 results that are known to zero the high bits.
446   FP16_ZEXT,
447 
448   /// This node is for VLIW targets and it is used to represent a vector
449   /// that is stored in consecutive registers with the same channel.
450   /// For example:
451   ///   |X  |Y|Z|W|
452   /// T0|v.x| | | |
453   /// T1|v.y| | | |
454   /// T2|v.z| | | |
455   /// T3|v.w| | | |
456   BUILD_VERTICAL_VECTOR,
457   /// Pointer to the start of the shader's constant data.
458   CONST_DATA_PTR,
459   INIT_EXEC,
460   INIT_EXEC_FROM_INPUT,
461   SENDMSG,
462   SENDMSGHALT,
463   INTERP_MOV,
464   INTERP_P1,
465   INTERP_P2,
466   PC_ADD_REL_OFFSET,
467   KILL,
468   DUMMY_CHAIN,
469   FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
470   STORE_MSKOR,
471   LOAD_CONSTANT,
472   TBUFFER_STORE_FORMAT,
473   TBUFFER_STORE_FORMAT_X3,
474   TBUFFER_STORE_FORMAT_D16,
475   TBUFFER_LOAD_FORMAT,
476   TBUFFER_LOAD_FORMAT_D16,
477   ATOMIC_CMP_SWAP,
478   ATOMIC_INC,
479   ATOMIC_DEC,
480   ATOMIC_LOAD_FADD,
481   ATOMIC_LOAD_FMIN,
482   ATOMIC_LOAD_FMAX,
483   BUFFER_LOAD,
484   BUFFER_LOAD_FORMAT,
485   BUFFER_LOAD_FORMAT_D16,
486   SBUFFER_LOAD,
487   BUFFER_STORE,
488   BUFFER_STORE_FORMAT,
489   BUFFER_STORE_FORMAT_D16,
490   BUFFER_ATOMIC_SWAP,
491   BUFFER_ATOMIC_ADD,
492   BUFFER_ATOMIC_SUB,
493   BUFFER_ATOMIC_SMIN,
494   BUFFER_ATOMIC_UMIN,
495   BUFFER_ATOMIC_SMAX,
496   BUFFER_ATOMIC_UMAX,
497   BUFFER_ATOMIC_AND,
498   BUFFER_ATOMIC_OR,
499   BUFFER_ATOMIC_XOR,
500   BUFFER_ATOMIC_CMPSWAP,
501 
502   LAST_AMDGPU_ISD_NUMBER
503 };
504 
505 
506 } // End namespace AMDGPUISD
507 
508 } // End namespace llvm
509 
510 #endif
511