1 //===-- NVPTXISelLowering.h - NVPTX DAG Lowering Interface ------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that NVPTX uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_LIB_TARGET_NVPTX_NVPTXISELLOWERING_H
16 #define LLVM_LIB_TARGET_NVPTX_NVPTXISELLOWERING_H
17 
18 #include "NVPTX.h"
19 #include "llvm/CodeGen/SelectionDAG.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 
22 namespace llvm {
23 namespace NVPTXISD {
24 enum NodeType : unsigned {
25   // Start the numbering from where ISD NodeType finishes.
26   FIRST_NUMBER = ISD::BUILTIN_OP_END,
27   Wrapper,
28   CALL,
29   RET_FLAG,
30   LOAD_PARAM,
31   DeclareParam,
32   DeclareScalarParam,
33   DeclareRetParam,
34   DeclareRet,
35   DeclareScalarRet,
36   PrintCall,
37   PrintConvergentCall,
38   PrintCallUni,
39   PrintConvergentCallUni,
40   CallArgBegin,
41   CallArg,
42   LastCallArg,
43   CallArgEnd,
44   CallVoid,
45   CallVal,
46   CallSymbol,
47   Prototype,
48   MoveParam,
49   PseudoUseParam,
50   RETURN,
51   CallSeqBegin,
52   CallSeqEnd,
53   CallPrototype,
54   ProxyReg,
55   FUN_SHFL_CLAMP,
56   FUN_SHFR_CLAMP,
57   MUL_WIDE_SIGNED,
58   MUL_WIDE_UNSIGNED,
59   IMAD,
60   SETP_F16X2,
61   Dummy,
62 
63   LoadV2 = ISD::FIRST_TARGET_MEMORY_OPCODE,
64   LoadV4,
65   LDGV2, // LDG.v2
66   LDGV4, // LDG.v4
67   LDUV2, // LDU.v2
68   LDUV4, // LDU.v4
69   StoreV2,
70   StoreV4,
71   LoadParam,
72   LoadParamV2,
73   LoadParamV4,
74   StoreParam,
75   StoreParamV2,
76   StoreParamV4,
77   StoreParamS32, // to sext and store a <32bit value, not used currently
78   StoreParamU32, // to zext and store a <32bit value, not used currently
79   StoreRetval,
80   StoreRetvalV2,
81   StoreRetvalV4,
82 
83   // Texture intrinsics
84   Tex1DFloatS32,
85   Tex1DFloatFloat,
86   Tex1DFloatFloatLevel,
87   Tex1DFloatFloatGrad,
88   Tex1DS32S32,
89   Tex1DS32Float,
90   Tex1DS32FloatLevel,
91   Tex1DS32FloatGrad,
92   Tex1DU32S32,
93   Tex1DU32Float,
94   Tex1DU32FloatLevel,
95   Tex1DU32FloatGrad,
96   Tex1DArrayFloatS32,
97   Tex1DArrayFloatFloat,
98   Tex1DArrayFloatFloatLevel,
99   Tex1DArrayFloatFloatGrad,
100   Tex1DArrayS32S32,
101   Tex1DArrayS32Float,
102   Tex1DArrayS32FloatLevel,
103   Tex1DArrayS32FloatGrad,
104   Tex1DArrayU32S32,
105   Tex1DArrayU32Float,
106   Tex1DArrayU32FloatLevel,
107   Tex1DArrayU32FloatGrad,
108   Tex2DFloatS32,
109   Tex2DFloatFloat,
110   Tex2DFloatFloatLevel,
111   Tex2DFloatFloatGrad,
112   Tex2DS32S32,
113   Tex2DS32Float,
114   Tex2DS32FloatLevel,
115   Tex2DS32FloatGrad,
116   Tex2DU32S32,
117   Tex2DU32Float,
118   Tex2DU32FloatLevel,
119   Tex2DU32FloatGrad,
120   Tex2DArrayFloatS32,
121   Tex2DArrayFloatFloat,
122   Tex2DArrayFloatFloatLevel,
123   Tex2DArrayFloatFloatGrad,
124   Tex2DArrayS32S32,
125   Tex2DArrayS32Float,
126   Tex2DArrayS32FloatLevel,
127   Tex2DArrayS32FloatGrad,
128   Tex2DArrayU32S32,
129   Tex2DArrayU32Float,
130   Tex2DArrayU32FloatLevel,
131   Tex2DArrayU32FloatGrad,
132   Tex3DFloatS32,
133   Tex3DFloatFloat,
134   Tex3DFloatFloatLevel,
135   Tex3DFloatFloatGrad,
136   Tex3DS32S32,
137   Tex3DS32Float,
138   Tex3DS32FloatLevel,
139   Tex3DS32FloatGrad,
140   Tex3DU32S32,
141   Tex3DU32Float,
142   Tex3DU32FloatLevel,
143   Tex3DU32FloatGrad,
144   TexCubeFloatFloat,
145   TexCubeFloatFloatLevel,
146   TexCubeS32Float,
147   TexCubeS32FloatLevel,
148   TexCubeU32Float,
149   TexCubeU32FloatLevel,
150   TexCubeArrayFloatFloat,
151   TexCubeArrayFloatFloatLevel,
152   TexCubeArrayS32Float,
153   TexCubeArrayS32FloatLevel,
154   TexCubeArrayU32Float,
155   TexCubeArrayU32FloatLevel,
156   Tld4R2DFloatFloat,
157   Tld4G2DFloatFloat,
158   Tld4B2DFloatFloat,
159   Tld4A2DFloatFloat,
160   Tld4R2DS64Float,
161   Tld4G2DS64Float,
162   Tld4B2DS64Float,
163   Tld4A2DS64Float,
164   Tld4R2DU64Float,
165   Tld4G2DU64Float,
166   Tld4B2DU64Float,
167   Tld4A2DU64Float,
168   TexUnified1DFloatS32,
169   TexUnified1DFloatFloat,
170   TexUnified1DFloatFloatLevel,
171   TexUnified1DFloatFloatGrad,
172   TexUnified1DS32S32,
173   TexUnified1DS32Float,
174   TexUnified1DS32FloatLevel,
175   TexUnified1DS32FloatGrad,
176   TexUnified1DU32S32,
177   TexUnified1DU32Float,
178   TexUnified1DU32FloatLevel,
179   TexUnified1DU32FloatGrad,
180   TexUnified1DArrayFloatS32,
181   TexUnified1DArrayFloatFloat,
182   TexUnified1DArrayFloatFloatLevel,
183   TexUnified1DArrayFloatFloatGrad,
184   TexUnified1DArrayS32S32,
185   TexUnified1DArrayS32Float,
186   TexUnified1DArrayS32FloatLevel,
187   TexUnified1DArrayS32FloatGrad,
188   TexUnified1DArrayU32S32,
189   TexUnified1DArrayU32Float,
190   TexUnified1DArrayU32FloatLevel,
191   TexUnified1DArrayU32FloatGrad,
192   TexUnified2DFloatS32,
193   TexUnified2DFloatFloat,
194   TexUnified2DFloatFloatLevel,
195   TexUnified2DFloatFloatGrad,
196   TexUnified2DS32S32,
197   TexUnified2DS32Float,
198   TexUnified2DS32FloatLevel,
199   TexUnified2DS32FloatGrad,
200   TexUnified2DU32S32,
201   TexUnified2DU32Float,
202   TexUnified2DU32FloatLevel,
203   TexUnified2DU32FloatGrad,
204   TexUnified2DArrayFloatS32,
205   TexUnified2DArrayFloatFloat,
206   TexUnified2DArrayFloatFloatLevel,
207   TexUnified2DArrayFloatFloatGrad,
208   TexUnified2DArrayS32S32,
209   TexUnified2DArrayS32Float,
210   TexUnified2DArrayS32FloatLevel,
211   TexUnified2DArrayS32FloatGrad,
212   TexUnified2DArrayU32S32,
213   TexUnified2DArrayU32Float,
214   TexUnified2DArrayU32FloatLevel,
215   TexUnified2DArrayU32FloatGrad,
216   TexUnified3DFloatS32,
217   TexUnified3DFloatFloat,
218   TexUnified3DFloatFloatLevel,
219   TexUnified3DFloatFloatGrad,
220   TexUnified3DS32S32,
221   TexUnified3DS32Float,
222   TexUnified3DS32FloatLevel,
223   TexUnified3DS32FloatGrad,
224   TexUnified3DU32S32,
225   TexUnified3DU32Float,
226   TexUnified3DU32FloatLevel,
227   TexUnified3DU32FloatGrad,
228   TexUnifiedCubeFloatFloat,
229   TexUnifiedCubeFloatFloatLevel,
230   TexUnifiedCubeS32Float,
231   TexUnifiedCubeS32FloatLevel,
232   TexUnifiedCubeU32Float,
233   TexUnifiedCubeU32FloatLevel,
234   TexUnifiedCubeArrayFloatFloat,
235   TexUnifiedCubeArrayFloatFloatLevel,
236   TexUnifiedCubeArrayS32Float,
237   TexUnifiedCubeArrayS32FloatLevel,
238   TexUnifiedCubeArrayU32Float,
239   TexUnifiedCubeArrayU32FloatLevel,
240   Tld4UnifiedR2DFloatFloat,
241   Tld4UnifiedG2DFloatFloat,
242   Tld4UnifiedB2DFloatFloat,
243   Tld4UnifiedA2DFloatFloat,
244   Tld4UnifiedR2DS64Float,
245   Tld4UnifiedG2DS64Float,
246   Tld4UnifiedB2DS64Float,
247   Tld4UnifiedA2DS64Float,
248   Tld4UnifiedR2DU64Float,
249   Tld4UnifiedG2DU64Float,
250   Tld4UnifiedB2DU64Float,
251   Tld4UnifiedA2DU64Float,
252 
253   // Surface intrinsics
254   Suld1DI8Clamp,
255   Suld1DI16Clamp,
256   Suld1DI32Clamp,
257   Suld1DI64Clamp,
258   Suld1DV2I8Clamp,
259   Suld1DV2I16Clamp,
260   Suld1DV2I32Clamp,
261   Suld1DV2I64Clamp,
262   Suld1DV4I8Clamp,
263   Suld1DV4I16Clamp,
264   Suld1DV4I32Clamp,
265 
266   Suld1DArrayI8Clamp,
267   Suld1DArrayI16Clamp,
268   Suld1DArrayI32Clamp,
269   Suld1DArrayI64Clamp,
270   Suld1DArrayV2I8Clamp,
271   Suld1DArrayV2I16Clamp,
272   Suld1DArrayV2I32Clamp,
273   Suld1DArrayV2I64Clamp,
274   Suld1DArrayV4I8Clamp,
275   Suld1DArrayV4I16Clamp,
276   Suld1DArrayV4I32Clamp,
277 
278   Suld2DI8Clamp,
279   Suld2DI16Clamp,
280   Suld2DI32Clamp,
281   Suld2DI64Clamp,
282   Suld2DV2I8Clamp,
283   Suld2DV2I16Clamp,
284   Suld2DV2I32Clamp,
285   Suld2DV2I64Clamp,
286   Suld2DV4I8Clamp,
287   Suld2DV4I16Clamp,
288   Suld2DV4I32Clamp,
289 
290   Suld2DArrayI8Clamp,
291   Suld2DArrayI16Clamp,
292   Suld2DArrayI32Clamp,
293   Suld2DArrayI64Clamp,
294   Suld2DArrayV2I8Clamp,
295   Suld2DArrayV2I16Clamp,
296   Suld2DArrayV2I32Clamp,
297   Suld2DArrayV2I64Clamp,
298   Suld2DArrayV4I8Clamp,
299   Suld2DArrayV4I16Clamp,
300   Suld2DArrayV4I32Clamp,
301 
302   Suld3DI8Clamp,
303   Suld3DI16Clamp,
304   Suld3DI32Clamp,
305   Suld3DI64Clamp,
306   Suld3DV2I8Clamp,
307   Suld3DV2I16Clamp,
308   Suld3DV2I32Clamp,
309   Suld3DV2I64Clamp,
310   Suld3DV4I8Clamp,
311   Suld3DV4I16Clamp,
312   Suld3DV4I32Clamp,
313 
314   Suld1DI8Trap,
315   Suld1DI16Trap,
316   Suld1DI32Trap,
317   Suld1DI64Trap,
318   Suld1DV2I8Trap,
319   Suld1DV2I16Trap,
320   Suld1DV2I32Trap,
321   Suld1DV2I64Trap,
322   Suld1DV4I8Trap,
323   Suld1DV4I16Trap,
324   Suld1DV4I32Trap,
325 
326   Suld1DArrayI8Trap,
327   Suld1DArrayI16Trap,
328   Suld1DArrayI32Trap,
329   Suld1DArrayI64Trap,
330   Suld1DArrayV2I8Trap,
331   Suld1DArrayV2I16Trap,
332   Suld1DArrayV2I32Trap,
333   Suld1DArrayV2I64Trap,
334   Suld1DArrayV4I8Trap,
335   Suld1DArrayV4I16Trap,
336   Suld1DArrayV4I32Trap,
337 
338   Suld2DI8Trap,
339   Suld2DI16Trap,
340   Suld2DI32Trap,
341   Suld2DI64Trap,
342   Suld2DV2I8Trap,
343   Suld2DV2I16Trap,
344   Suld2DV2I32Trap,
345   Suld2DV2I64Trap,
346   Suld2DV4I8Trap,
347   Suld2DV4I16Trap,
348   Suld2DV4I32Trap,
349 
350   Suld2DArrayI8Trap,
351   Suld2DArrayI16Trap,
352   Suld2DArrayI32Trap,
353   Suld2DArrayI64Trap,
354   Suld2DArrayV2I8Trap,
355   Suld2DArrayV2I16Trap,
356   Suld2DArrayV2I32Trap,
357   Suld2DArrayV2I64Trap,
358   Suld2DArrayV4I8Trap,
359   Suld2DArrayV4I16Trap,
360   Suld2DArrayV4I32Trap,
361 
362   Suld3DI8Trap,
363   Suld3DI16Trap,
364   Suld3DI32Trap,
365   Suld3DI64Trap,
366   Suld3DV2I8Trap,
367   Suld3DV2I16Trap,
368   Suld3DV2I32Trap,
369   Suld3DV2I64Trap,
370   Suld3DV4I8Trap,
371   Suld3DV4I16Trap,
372   Suld3DV4I32Trap,
373 
374   Suld1DI8Zero,
375   Suld1DI16Zero,
376   Suld1DI32Zero,
377   Suld1DI64Zero,
378   Suld1DV2I8Zero,
379   Suld1DV2I16Zero,
380   Suld1DV2I32Zero,
381   Suld1DV2I64Zero,
382   Suld1DV4I8Zero,
383   Suld1DV4I16Zero,
384   Suld1DV4I32Zero,
385 
386   Suld1DArrayI8Zero,
387   Suld1DArrayI16Zero,
388   Suld1DArrayI32Zero,
389   Suld1DArrayI64Zero,
390   Suld1DArrayV2I8Zero,
391   Suld1DArrayV2I16Zero,
392   Suld1DArrayV2I32Zero,
393   Suld1DArrayV2I64Zero,
394   Suld1DArrayV4I8Zero,
395   Suld1DArrayV4I16Zero,
396   Suld1DArrayV4I32Zero,
397 
398   Suld2DI8Zero,
399   Suld2DI16Zero,
400   Suld2DI32Zero,
401   Suld2DI64Zero,
402   Suld2DV2I8Zero,
403   Suld2DV2I16Zero,
404   Suld2DV2I32Zero,
405   Suld2DV2I64Zero,
406   Suld2DV4I8Zero,
407   Suld2DV4I16Zero,
408   Suld2DV4I32Zero,
409 
410   Suld2DArrayI8Zero,
411   Suld2DArrayI16Zero,
412   Suld2DArrayI32Zero,
413   Suld2DArrayI64Zero,
414   Suld2DArrayV2I8Zero,
415   Suld2DArrayV2I16Zero,
416   Suld2DArrayV2I32Zero,
417   Suld2DArrayV2I64Zero,
418   Suld2DArrayV4I8Zero,
419   Suld2DArrayV4I16Zero,
420   Suld2DArrayV4I32Zero,
421 
422   Suld3DI8Zero,
423   Suld3DI16Zero,
424   Suld3DI32Zero,
425   Suld3DI64Zero,
426   Suld3DV2I8Zero,
427   Suld3DV2I16Zero,
428   Suld3DV2I32Zero,
429   Suld3DV2I64Zero,
430   Suld3DV4I8Zero,
431   Suld3DV4I16Zero,
432   Suld3DV4I32Zero
433 };
434 }
435 
436 class NVPTXSubtarget;
437 
438 //===--------------------------------------------------------------------===//
439 // TargetLowering Implementation
440 //===--------------------------------------------------------------------===//
441 class NVPTXTargetLowering : public TargetLowering {
442 public:
443   explicit NVPTXTargetLowering(const NVPTXTargetMachine &TM,
444                                const NVPTXSubtarget &STI);
445   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
446 
447   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
448 
449   const char *getTargetNodeName(unsigned Opcode) const override;
450 
451   bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
452                           MachineFunction &MF,
453                           unsigned Intrinsic) const override;
454 
455   /// isLegalAddressingMode - Return true if the addressing mode represented
456   /// by AM is legal for this target, for a load/store of the specified type
457   /// Used to guide target specific optimizations, like loop strength
458   /// reduction (LoopStrengthReduce.cpp) and memory optimization for
459   /// address mode (CodeGenPrepare.cpp)
460   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
461                              unsigned AS,
462                              Instruction *I = nullptr) const override;
463 
isTruncateFree(Type * SrcTy,Type * DstTy)464   bool isTruncateFree(Type *SrcTy, Type *DstTy) const override {
465     // Truncating 64-bit to 32-bit is free in SASS.
466     if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
467       return false;
468     return SrcTy->getPrimitiveSizeInBits() == 64 &&
469            DstTy->getPrimitiveSizeInBits() == 32;
470   }
471 
getSetCCResultType(const DataLayout & DL,LLVMContext & Ctx,EVT VT)472   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
473                          EVT VT) const override {
474     if (VT.isVector())
475       return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
476     return MVT::i1;
477   }
478 
479   ConstraintType getConstraintType(StringRef Constraint) const override;
480   std::pair<unsigned, const TargetRegisterClass *>
481   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
482                                StringRef Constraint, MVT VT) const override;
483 
484   SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
485                                bool isVarArg,
486                                const SmallVectorImpl<ISD::InputArg> &Ins,
487                                const SDLoc &dl, SelectionDAG &DAG,
488                                SmallVectorImpl<SDValue> &InVals) const override;
489 
490   SDValue LowerCall(CallLoweringInfo &CLI,
491                     SmallVectorImpl<SDValue> &InVals) const override;
492 
493   std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
494                            const SmallVectorImpl<ISD::OutputArg> &,
495                            unsigned retAlignment,
496                            ImmutableCallSite CS) const;
497 
498   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
499                       const SmallVectorImpl<ISD::OutputArg> &Outs,
500                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
501                       SelectionDAG &DAG) const override;
502 
503   void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
504                                     std::vector<SDValue> &Ops,
505                                     SelectionDAG &DAG) const override;
506 
507   const NVPTXTargetMachine *nvTM;
508 
509   // PTX always uses 32-bit shift amounts
getScalarShiftAmountTy(const DataLayout &,EVT)510   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
511     return MVT::i32;
512   }
513 
514   TargetLoweringBase::LegalizeTypeAction
515   getPreferredVectorAction(MVT VT) const override;
516 
517   // Get the degree of precision we want from 32-bit floating point division
518   // operations.
519   //
520   //  0 - Use ptx div.approx
521   //  1 - Use ptx.div.full (approximate, but less so than div.approx)
522   //  2 - Use IEEE-compliant div instructions, if available.
523   int getDivF32Level() const;
524 
525   // Get whether we should use a precise or approximate 32-bit floating point
526   // sqrt instruction.
527   bool usePrecSqrtF32() const;
528 
529   // Get whether we should use instructions that flush floating-point denormals
530   // to sign-preserving zero.
531   bool useF32FTZ(const MachineFunction &MF) const;
532 
533   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
534                           int &ExtraSteps, bool &UseOneConst,
535                           bool Reciprocal) const override;
536 
combineRepeatedFPDivisors()537   unsigned combineRepeatedFPDivisors() const override { return 2; }
538 
539   bool allowFMA(MachineFunction &MF, CodeGenOpt::Level OptLevel) const;
540   bool allowUnsafeFPMath(MachineFunction &MF) const;
541 
isFMAFasterThanFMulAndFAdd(EVT)542   bool isFMAFasterThanFMulAndFAdd(EVT) const override { return true; }
543 
enableAggressiveFMAFusion(EVT VT)544   bool enableAggressiveFMAFusion(EVT VT) const override { return true; }
545 
546   // The default is to transform llvm.ctlz(x, false) (where false indicates that
547   // x == 0 is not undefined behavior) into a branch that checks whether x is 0
548   // and avoids calling ctlz in that case.  We have a dedicated ctlz
549   // instruction, so we say that ctlz is cheap to speculate.
isCheapToSpeculateCtlz()550   bool isCheapToSpeculateCtlz() const override { return true; }
551 
552 private:
553   const NVPTXSubtarget &STI; // cache the subtarget here
554   SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
555 
556   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
557   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
558   SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
559 
560   SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
561   SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const;
562 
563   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
564   SDValue LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const;
565   SDValue LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const;
566 
567   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
568   SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
569 
570   SDValue LowerSelect(SDValue Op, SelectionDAG &DAG) const;
571 
572   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
573                           SelectionDAG &DAG) const override;
574   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
575 
576   unsigned getArgumentAlignment(SDValue Callee, ImmutableCallSite CS, Type *Ty,
577                                 unsigned Idx, const DataLayout &DL) const;
578 };
579 } // namespace llvm
580 
581 #endif
582