1 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI DAG Lowering interface definition 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H 15 #define LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H 16 17 #include "AMDGPUISelLowering.h" 18 #include "AMDGPUArgumentUsageInfo.h" 19 #include "SIInstrInfo.h" 20 21 namespace llvm { 22 23 class SITargetLowering final : public AMDGPUTargetLowering { 24 private: 25 const GCNSubtarget *Subtarget; 26 27 public: 28 MVT getRegisterTypeForCallingConv(LLVMContext &Context, 29 CallingConv::ID CC, 30 EVT VT) const override; 31 unsigned getNumRegistersForCallingConv(LLVMContext &Context, 32 CallingConv::ID CC, 33 EVT VT) const override; 34 35 unsigned getVectorTypeBreakdownForCallingConv( 36 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, 37 unsigned &NumIntermediates, MVT &RegisterVT) const override; 38 39 private: 40 SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL, 41 SDValue Chain, uint64_t Offset) const; 42 SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const; 43 SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT, 44 const SDLoc &SL, SDValue Chain, 45 uint64_t Offset, unsigned Align, bool Signed, 46 const ISD::InputArg *Arg = nullptr) const; 47 48 SDValue lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA, 49 const SDLoc &SL, SDValue Chain, 50 const ISD::InputArg &Arg) const; 51 SDValue getPreloadedValue(SelectionDAG &DAG, 52 const SIMachineFunctionInfo &MFI, 53 EVT VT, 54 AMDGPUFunctionArgInfo::PreloadedValue) const; 55 56 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, 57 SelectionDAG &DAG) const override; 58 SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op, 59 MVT VT, unsigned Offset) const; 60 SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr, 61 SelectionDAG &DAG) const; 62 SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset, 63 SDValue GLC, SelectionDAG &DAG) const; 64 65 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; 66 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const; 67 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; 68 69 // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset 70 // (the offset that is included in bounds checking and swizzling, to be split 71 // between the instruction's voffset and immoffset fields) and soffset (the 72 // offset that is excluded from bounds checking and swizzling, to go in the 73 // instruction's soffset field). This function takes the first kind of 74 // offset and figures out how to split it between voffset and immoffset. 75 std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset, 76 SelectionDAG &DAG) const; 77 78 SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const; 79 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; 80 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const; 81 SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const; 82 SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const; 83 SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const; 84 SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const; 85 SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const; 86 SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const; 87 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, bool Signed) const; 88 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const; 89 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const; 90 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const; 91 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const; 92 93 SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M, 94 SelectionDAG &DAG, ArrayRef<SDValue> Ops, 95 bool IsIntrinsic = false) const; 96 97 // Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to 98 // dwordx4 if on SI. 99 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 100 ArrayRef<SDValue> Ops, EVT MemVT, 101 MachineMemOperand *MMO, SelectionDAG &DAG) const; 102 103 SDValue handleD16VData(SDValue VData, SelectionDAG &DAG) const; 104 105 /// Converts \p Op, which must be of floating point type, to the 106 /// floating point type \p VT, by either extending or truncating it. 107 SDValue getFPExtOrFPTrunc(SelectionDAG &DAG, 108 SDValue Op, 109 const SDLoc &DL, 110 EVT VT) const; 111 112 SDValue convertArgType( 113 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val, 114 bool Signed, const ISD::InputArg *Arg = nullptr) const; 115 116 /// Custom lowering for ISD::FP_ROUND for MVT::f16. 117 SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const; 118 SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const; 119 120 SDValue getSegmentAperture(unsigned AS, const SDLoc &DL, 121 SelectionDAG &DAG) const; 122 123 SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const; 124 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; 125 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; 126 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; 127 SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const; 128 SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const; 129 130 SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const; 131 132 SDValue performUCharToFloatCombine(SDNode *N, 133 DAGCombinerInfo &DCI) const; 134 SDValue performSHLPtrCombine(SDNode *N, 135 unsigned AS, 136 EVT MemVT, 137 DAGCombinerInfo &DCI) const; 138 139 SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const; 140 141 SDValue splitBinaryBitConstantOp(DAGCombinerInfo &DCI, const SDLoc &SL, 142 unsigned Opc, SDValue LHS, 143 const ConstantSDNode *CRHS) const; 144 145 SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const; 146 SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const; 147 SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const; 148 SDValue performZeroExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const; 149 SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const; 150 SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const; 151 SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT, 152 const APFloat &C) const; 153 SDValue performFCanonicalizeCombine(SDNode *N, DAGCombinerInfo &DCI) const; 154 155 SDValue performFPMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL, 156 SDValue Op0, SDValue Op1) const; 157 SDValue performIntMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL, 158 SDValue Op0, SDValue Op1, bool Signed) const; 159 SDValue performMinMaxCombine(SDNode *N, DAGCombinerInfo &DCI) const; 160 SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const; 161 SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const; 162 SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const; 163 SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const; 164 165 SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const; 166 unsigned getFusedOpcode(const SelectionDAG &DAG, 167 const SDNode *N0, const SDNode *N1) const; 168 SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const; 169 SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const; 170 SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const; 171 SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const; 172 SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const; 173 SDValue performFMACombine(SDNode *N, DAGCombinerInfo &DCI) const; 174 SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const; 175 SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const; 176 SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const; 177 SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const; 178 179 bool isLegalFlatAddressingMode(const AddrMode &AM) const; 180 bool isLegalMUBUFAddressingMode(const AddrMode &AM) const; 181 182 unsigned isCFIntrinsic(const SDNode *Intr) const; 183 184 /// \returns True if fixup needs to be emitted for given global value \p GV, 185 /// false otherwise. 186 bool shouldEmitFixup(const GlobalValue *GV) const; 187 188 /// \returns True if GOT relocation needs to be emitted for given global value 189 /// \p GV, false otherwise. 190 bool shouldEmitGOTReloc(const GlobalValue *GV) const; 191 192 /// \returns True if PC-relative relocation needs to be emitted for given 193 /// global value \p GV, false otherwise. 194 bool shouldEmitPCReloc(const GlobalValue *GV) const; 195 196 // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the 197 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array 198 // pointed to by Offsets. 199 void setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG, 200 SDValue *Offsets, unsigned Align = 4) const; 201 202 // Handle 8 bit and 16 bit buffer loads 203 SDValue handleByteShortBufferLoads(SelectionDAG &DAG, EVT LoadVT, SDLoc DL, 204 ArrayRef<SDValue> Ops, MemSDNode *M) const; 205 206 // Handle 8 bit and 16 bit buffer stores 207 SDValue handleByteShortBufferStores(SelectionDAG &DAG, EVT VDataType, 208 SDLoc DL, SDValue Ops[], 209 MemSDNode *M) const; 210 211 public: 212 SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI); 213 214 const GCNSubtarget *getSubtarget() const; 215 216 bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const override; 217 218 bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override; 219 220 bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, 221 MachineFunction &MF, 222 unsigned IntrinsicID) const override; 223 224 bool getAddrModeArguments(IntrinsicInst * /*I*/, 225 SmallVectorImpl<Value*> &/*Ops*/, 226 Type *&/*AccessTy*/) const override; 227 228 bool isLegalGlobalAddressingMode(const AddrMode &AM) const; 229 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, 230 unsigned AS, 231 Instruction *I = nullptr) const override; 232 233 bool canMergeStoresTo(unsigned AS, EVT MemVT, 234 const SelectionDAG &DAG) const override; 235 236 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, 237 unsigned Align, 238 bool *IsFast) const override; 239 240 EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, 241 unsigned SrcAlign, bool IsMemset, 242 bool ZeroMemset, 243 bool MemcpyStrSrc, 244 MachineFunction &MF) const override; 245 246 bool isMemOpUniform(const SDNode *N) const; 247 bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const; 248 bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; 249 bool isCheapAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; 250 251 TargetLoweringBase::LegalizeTypeAction 252 getPreferredVectorAction(MVT VT) const override; 253 254 bool shouldConvertConstantLoadToIntImm(const APInt &Imm, 255 Type *Ty) const override; 256 257 bool isTypeDesirableForOp(unsigned Op, EVT VT) const override; 258 259 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; 260 261 bool supportSplitCSR(MachineFunction *MF) const override; 262 void initializeSplitCSR(MachineBasicBlock *Entry) const override; 263 void insertCopiesSplitCSR( 264 MachineBasicBlock *Entry, 265 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override; 266 267 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, 268 bool isVarArg, 269 const SmallVectorImpl<ISD::InputArg> &Ins, 270 const SDLoc &DL, SelectionDAG &DAG, 271 SmallVectorImpl<SDValue> &InVals) const override; 272 273 bool CanLowerReturn(CallingConv::ID CallConv, 274 MachineFunction &MF, bool isVarArg, 275 const SmallVectorImpl<ISD::OutputArg> &Outs, 276 LLVMContext &Context) const override; 277 278 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 279 const SmallVectorImpl<ISD::OutputArg> &Outs, 280 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, 281 SelectionDAG &DAG) const override; 282 283 void passSpecialInputs( 284 CallLoweringInfo &CLI, 285 CCState &CCInfo, 286 const SIMachineFunctionInfo &Info, 287 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, 288 SmallVectorImpl<SDValue> &MemOpChains, 289 SDValue Chain) const; 290 291 SDValue LowerCallResult(SDValue Chain, SDValue InFlag, 292 CallingConv::ID CallConv, bool isVarArg, 293 const SmallVectorImpl<ISD::InputArg> &Ins, 294 const SDLoc &DL, SelectionDAG &DAG, 295 SmallVectorImpl<SDValue> &InVals, bool isThisReturn, 296 SDValue ThisVal) const; 297 298 bool mayBeEmittedAsTailCall(const CallInst *) const override; 299 300 bool isEligibleForTailCallOptimization( 301 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, 302 const SmallVectorImpl<ISD::OutputArg> &Outs, 303 const SmallVectorImpl<SDValue> &OutVals, 304 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const; 305 306 SDValue LowerCall(CallLoweringInfo &CLI, 307 SmallVectorImpl<SDValue> &InVals) const override; 308 309 unsigned getRegisterByName(const char* RegName, EVT VT, 310 SelectionDAG &DAG) const override; 311 312 MachineBasicBlock *splitKillBlock(MachineInstr &MI, 313 MachineBasicBlock *BB) const; 314 315 MachineBasicBlock * 316 EmitInstrWithCustomInserter(MachineInstr &MI, 317 MachineBasicBlock *BB) const override; 318 319 bool hasBitPreservingFPLogic(EVT VT) const override; 320 bool enableAggressiveFMAFusion(EVT VT) const override; 321 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, 322 EVT VT) const override; 323 MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override; 324 bool isFMAFasterThanFMulAndFAdd(EVT VT) const override; 325 SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const; 326 SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const; 327 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; 328 329 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, 330 SelectionDAG &DAG) const override; 331 332 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; 333 SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override; 334 void AdjustInstrPostInstrSelection(MachineInstr &MI, 335 SDNode *Node) const override; 336 337 SDNode *legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const; 338 339 MachineSDNode *wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL, 340 SDValue Ptr) const; 341 MachineSDNode *buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, 342 uint32_t RsrcDword1, uint64_t RsrcDword2And3) const; 343 std::pair<unsigned, const TargetRegisterClass *> 344 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 345 StringRef Constraint, MVT VT) const override; 346 ConstraintType getConstraintType(StringRef Constraint) const override; 347 SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL, 348 SDValue V) const; 349 350 void finalizeLowering(MachineFunction &MF) const override; 351 352 void computeKnownBitsForFrameIndex(const SDValue Op, 353 KnownBits &Known, 354 const APInt &DemandedElts, 355 const SelectionDAG &DAG, 356 unsigned Depth = 0) const override; 357 358 bool isSDNodeSourceOfDivergence(const SDNode *N, 359 FunctionLoweringInfo *FLI, LegacyDivergenceAnalysis *DA) const override; 360 361 bool isCanonicalized(SelectionDAG &DAG, SDValue Op, 362 unsigned MaxDepth = 5) const; 363 bool denormalsEnabledForType(EVT VT) const; 364 365 bool isKnownNeverNaNForTargetNode(SDValue Op, 366 const SelectionDAG &DAG, 367 bool SNaN = false, 368 unsigned Depth = 0) const override; 369 AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override; 370 }; 371 372 } // End namespace llvm 373 374 #endif 375