Lines Matching refs:SL
1280 const SDLoc &SL, in lowerKernArgParameterPtr() argument
1295 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL, in lowerKernArgParameterPtr()
1298 return DAG.getObjectPtrOffset(SL, BasePtr, Offset); in lowerKernArgParameterPtr()
1302 const SDLoc &SL) const { in getImplicitArgPtr()
1305 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset); in getImplicitArgPtr()
1309 const SDLoc &SL, SDValue Val, in convertArgType() argument
1315 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT)); in convertArgType()
1319 Val = getFPExtOrFPTrunc(DAG, Val, SL, VT); in convertArgType()
1321 Val = DAG.getSExtOrTrunc(Val, SL, VT); in convertArgType()
1323 Val = DAG.getZExtOrTrunc(Val, SL, VT); in convertArgType()
1330 const SDLoc &SL, SDValue Chain, in lowerKernargMemParameter() argument
1349 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset); in lowerKernargMemParameter()
1350 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4, in lowerKernargMemParameter()
1354 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
1355 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
1357 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract); in lowerKernargMemParameter()
1358 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal); in lowerKernargMemParameter()
1359 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg); in lowerKernargMemParameter()
1362 return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL); in lowerKernargMemParameter()
1365 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset); in lowerKernargMemParameter()
1366 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Align, in lowerKernargMemParameter()
1370 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg); in lowerKernargMemParameter()
1371 return DAG.getMergeValues({ Val, Load.getValue(1) }, SL); in lowerKernargMemParameter()
1375 const SDLoc &SL, SDValue Chain, in lowerStackParameter() argument
1417 ExtType, SL, VA.getLocVT(), Chain, FIN, in lowerStackParameter()
3588 SDLoc SL(Op); in splitUnaryVectorOp() local
3589 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, in splitUnaryVectorOp()
3591 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, in splitUnaryVectorOp()
3610 SDLoc SL(Op); in splitBinaryVectorOp() local
3612 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, in splitBinaryVectorOp()
3614 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, in splitBinaryVectorOp()
3796 SDLoc SL(N); in lowerFCMPIntrinsic() local
3799 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
3800 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
3805 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src0, in lowerFCMPIntrinsic()
3829 SDLoc SL(N); in ReplaceNodeResults() local
3830 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
3832 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
3841 SDLoc SL(N); in ReplaceNodeResults() local
3855 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1)); in ReplaceNodeResults()
3857 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
3858 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
3875 SDLoc SL(N); in ReplaceNodeResults() local
3878 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1)); in ReplaceNodeResults()
3879 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2)); in ReplaceNodeResults()
3883 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
3884 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
3888 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT, in ReplaceNodeResults()
3892 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect); in ReplaceNodeResults()
3893 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect)); in ReplaceNodeResults()
3900 SDLoc SL(N); in ReplaceNodeResults() local
3901 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
3903 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
3905 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
3906 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
3913 SDLoc SL(N); in ReplaceNodeResults() local
3914 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
3916 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
3918 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
3919 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
4153 SDLoc SL(Op); in lowerTRAP() local
4158 return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); in lowerTRAP()
4167 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, in lowerTRAP()
4171 DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMTrap, SL, MVT::i16), in lowerTRAP()
4175 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTRAP()
4179 SDLoc SL(Op); in lowerDEBUGTRAP() local
4196 DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16) in lowerDEBUGTRAP()
4198 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
4252 SDLoc SL(Op); in lowerADDRSPACECAST() local
4256 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
4268 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
4269 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
4270 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
4272 return DAG.getNode(ISD::SELECT, SL, MVT::i32, in lowerADDRSPACECAST()
4284 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
4287 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
4289 SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG); in lowerADDRSPACECAST()
4291 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
4293 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, in lowerADDRSPACECAST()
4294 DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr), in lowerADDRSPACECAST()
4303 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); in lowerADDRSPACECAST()
4323 SDLoc SL(Op); in lowerINSERT_VECTOR_ELT() local
4327 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
4329 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
4330 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4331 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
4332 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4334 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
4335 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
4339 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
4341 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
4342 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4344 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
4347 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
4348 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
4350 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat); in lowerINSERT_VECTOR_ELT()
4361 Val = DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal); in lowerINSERT_VECTOR_ELT()
4364 SDValue ExtVal = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Val); in lowerINSERT_VECTOR_ELT()
4367 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
4370 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
4372 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerINSERT_VECTOR_ELT()
4373 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT, in lowerINSERT_VECTOR_ELT()
4374 DAG.getConstant(0xffff, SL, IntVT), in lowerINSERT_VECTOR_ELT()
4377 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal); in lowerINSERT_VECTOR_ELT()
4378 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT, in lowerINSERT_VECTOR_ELT()
4379 DAG.getNOT(SL, BFM, IntVT), BCVec); in lowerINSERT_VECTOR_ELT()
4381 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS); in lowerINSERT_VECTOR_ELT()
4382 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI); in lowerINSERT_VECTOR_ELT()
4387 SDLoc SL(Op); in lowerEXTRACT_VECTOR_ELT() local
4410 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
4413 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
4415 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerEXTRACT_VECTOR_ELT()
4416 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx); in lowerEXTRACT_VECTOR_ELT()
4419 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
4420 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result); in lowerEXTRACT_VECTOR_ELT()
4423 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT); in lowerEXTRACT_VECTOR_ELT()
4428 SDLoc SL(Op); in lowerBUILD_VECTOR() local
4436 SDValue Lo = DAG.getBuildVector(HalfVT, SL, in lowerBUILD_VECTOR()
4438 SDValue Hi = DAG.getBuildVector(HalfVT, SL, in lowerBUILD_VECTOR()
4441 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4442 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
4444 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi }); in lowerBUILD_VECTOR()
4445 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
4456 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
4457 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4458 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo); in lowerBUILD_VECTOR()
4461 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
4462 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
4464 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
4465 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
4467 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi); in lowerBUILD_VECTOR()
4469 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
4470 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4472 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
4473 return DAG.getNode(ISD::BITCAST, SL, VT, Or); in lowerBUILD_VECTOR()
4582 SDLoc SL(Op); in lowerImplicitZextParam() local
4583 SDValue Param = lowerKernargMemParameter(DAG, MVT::i32, MVT::i32, SL, in lowerImplicitZextParam()
4586 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
6349 const SDLoc &SL, EVT VT) { in getLoadExtOrTrunc() argument
6351 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op); in getLoadExtOrTrunc()
6355 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
6357 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
6359 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
6387 SDLoc SL(Ld); in widenLoad() local
6395 MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
6412 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
6416 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT); in widenLoad()
6428 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT); in widenLoad()
6432 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt); in widenLoad()
6434 return DAG.getMergeValues({ Cvt, NewLoad.getValue(1) }, SL); in widenLoad()
6596 SDLoc SL(Op); in lowerFastUnsafeFDIV() local
6621 return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0)); in lowerFastUnsafeFDIV()
6624 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
6630 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in lowerFastUnsafeFDIV()
6631 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS); in lowerFastUnsafeFDIV()
6639 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
6640 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags); in lowerFastUnsafeFDIV()
6646 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPBinOp() argument
6649 return DAG.getNode(Opcode, SL, VT, A, B); in getFPBinOp()
6662 return DAG.getNode(Opcode, SL, VTList, GlueChain.getValue(1), A, B, in getFPBinOp()
6666 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPTernOp() argument
6670 return DAG.getNode(Opcode, SL, VT, A, B, C); in getFPTernOp()
6683 return DAG.getNode(Opcode, SL, VTList, GlueChain.getValue(1), A, B, C, in getFPTernOp()
6691 SDLoc SL(Op); in LowerFDIV16() local
6695 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
6696 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
6698 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
6699 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
6701 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
6702 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
6704 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
6709 SDLoc SL(Op); in lowerFDIV_FAST() local
6713 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS); in lowerFDIV_FAST()
6716 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
6719 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
6721 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
6726 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT); in lowerFDIV_FAST()
6728 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One); in lowerFDIV_FAST()
6731 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3); in lowerFDIV_FAST()
6734 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1); in lowerFDIV_FAST()
6736 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0); in lowerFDIV_FAST()
6738 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul); in lowerFDIV_FAST()
6745 SDLoc SL(Op); in LowerFDIV32() local
6749 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
6753 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
6755 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
6759 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
6761 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
6768 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i16); in LowerFDIV32()
6773 SL, MVT::i32); in LowerFDIV32()
6774 SDValue EnableDenorm = DAG.getNode(AMDGPUISD::SETREG, SL, BindParamVTs, in LowerFDIV32()
6783 NegDivScale0 = DAG.getMergeValues(Ops, SL); in LowerFDIV32()
6786 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
6789 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
6792 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
6795 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
6798 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA,SL, MVT::f32, Fma2, Fma1, Mul, Fma2); in LowerFDIV32()
6800 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
6805 DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
6806 SDValue DisableDenorm = DAG.getNode(AMDGPUISD::SETREG, SL, MVT::Other, in LowerFDIV32()
6812 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
6818 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
6821 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS); in LowerFDIV32()
6828 SDLoc SL(Op); in LowerFDIV64() local
6832 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
6836 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X); in LowerFDIV64()
6838 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
6840 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
6842 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
6844 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
6846 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
6848 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X); in LowerFDIV64()
6850 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
6851 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
6853 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
6862 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
6865 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
6866 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
6867 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
6868 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
6870 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
6871 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
6874 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
6876 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
6878 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
6879 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
6880 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
6885 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
6888 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
7114 SDLoc SL(N); in performSHLPtrCombine() local
7117 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1); in performSHLPtrCombine()
7118 SDValue COffset = DAG.getConstant(Offset, SL, MVT::i32); in performSHLPtrCombine()
7125 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags); in performSHLPtrCombine()
7132 SDLoc SL(N); in performMemSDNodeCombine() local
7162 const SDLoc &SL, in splitBinaryBitConstantOp() argument
7176 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi); in splitBinaryBitConstantOp()
7296 SDLoc SL(N); in performAndCombine() local
7297 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
7299 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
7300 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
7302 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE, in performAndCombine()
7552 SDLoc SL(N); in performOrCombine() local
7555 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
7560 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
7562 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
7872 SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const { in getCanonicalConstantFP() argument
7875 return DAG.getConstantFP(0.0, SL, VT); in getCanonicalConstantFP()
7882 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
7890 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
7894 return DAG.getConstantFP(C, SL, VT); in getCanonicalConstantFP()
7928 SDLoc SL(N); in performFCanonicalizeCombine() local
7938 NewElts[I] = getCanonicalConstantFP(DAG, SL, EltVT, in performFCanonicalizeCombine()
7944 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op); in performFCanonicalizeCombine()
7954 NewElts[1]: DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
7959 NewElts[0] : DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
7962 return DAG.getBuildVector(VT, SL, NewElts); in performFCanonicalizeCombine()
7976 SDLoc SL(N); in performFCanonicalizeCombine() local
7977 SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT, in performFCanonicalizeCombine()
7979 SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF()); in performFCanonicalizeCombine()
7982 return DAG.getNode(N0.getOpcode(), SL, VT, Canon0, Canon1); in performFCanonicalizeCombine()
8011 SelectionDAG &DAG, const SDLoc &SL, in performIntMed3ImmCombine() argument
8032 return DAG.getNode(Med3Opc, SL, VT, in performIntMed3ImmCombine()
8040 SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0)); in performIntMed3ImmCombine()
8041 SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1)); in performIntMed3ImmCombine()
8042 SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1); in performIntMed3ImmCombine()
8044 SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3); in performIntMed3ImmCombine()
8045 return DAG.getNode(ISD::TRUNCATE, SL, VT, Med3); in performIntMed3ImmCombine()
8061 const SDLoc &SL, in performFPMed3ImmCombine() argument
8084 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0)); in performFPMed3ImmCombine()
8103 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), in performFPMed3ImmCombine()
8200 SDLoc SL(N); in performFMed3Combine() local
8210 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2); in performFMed3Combine()
8228 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0); in performFMed3Combine()
8253 SDLoc SL(N); in performExtractVectorEltCombine() local
8256 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in performExtractVectorEltCombine()
8258 return DAG.getNode(Vec.getOpcode(), SL, EltVT, Elt); in performExtractVectorEltCombine()
8267 SDLoc SL(N); in performExtractVectorEltCombine() local
8288 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in performExtractVectorEltCombine()
8290 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in performExtractVectorEltCombine()
8295 return DAG.getNode(Opc, SL, EltVT, Elt0, Elt1, Vec->getFlags()); in performExtractVectorEltCombine()
8310 SDLoc SL(N); in performExtractVectorEltCombine() local
8315 SDValue IC = DAG.getConstant(I, SL, IdxVT); in performExtractVectorEltCombine()
8316 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); in performExtractVectorEltCombine()
8320 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ); in performExtractVectorEltCombine()
8343 SDLoc SL(N); in performExtractVectorEltCombine() local
8345 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec); in performExtractVectorEltCombine()
8348 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
8349 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
8351 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
8352 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
8355 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, EltVT.changeTypeToInteger(), Srl); in performExtractVectorEltCombine()
8357 return DAG.getNode(ISD::BITCAST, SL, EltVT, Trunc); in performExtractVectorEltCombine()
8384 SDLoc SL(N); in performInsertVectorEltCombine() local
8390 SDValue IC = DAG.getConstant(I, SL, IdxVT); in performInsertVectorEltCombine()
8391 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); in performInsertVectorEltCombine()
8392 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ); in performInsertVectorEltCombine()
8396 return DAG.getBuildVector(VecVT, SL, Ops); in performInsertVectorEltCombine()
8421 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, in getMad64_32() argument
8427 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2); in getMad64_32()
8428 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad); in getMad64_32()
8435 SDLoc SL(N); in performAddCombine() local
8453 MulLHS = DAG.getZExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
8454 MulRHS = DAG.getZExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
8455 AddRHS = DAG.getZExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
8456 return getMad64_32(DAG, SL, VT, MulLHS, MulRHS, AddRHS, false); in performAddCombine()
8460 MulLHS = DAG.getSExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
8461 MulRHS = DAG.getSExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
8462 AddRHS = DAG.getSExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
8463 return getMad64_32(DAG, SL, VT, MulLHS, MulRHS, AddRHS, true); in performAddCombine()
8489 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
8491 return DAG.getNode(Opc, SL, VTList, Args); in performAddCombine()
8512 SDLoc SL(N); in performSubCombine() local
8564 SDLoc SL(N); in performFAddCombine() local
8577 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
8578 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS); in performFAddCombine()
8589 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
8590 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS); in performFAddCombine()
8604 SDLoc SL(N); in performFSubCombine() local
8621 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFSubCombine()
8622 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFSubCombine()
8624 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS); in performFSubCombine()
8636 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT); in performFSubCombine()
8637 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS); in performFSubCombine()
8649 SDLoc SL(N); in performFMACombine() local
8713 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
8714 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
8723 SDLoc SL(N); in performSetCCCombine() local
8750 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8751 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
8776 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8777 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
8808 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8809 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
8819 SDLoc SL(N); in performCvtF32UByteNCombine() local
8840 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + SrcOffset / 8, SL, in performCvtF32UByteNCombine()
8991 SDLoc SL(N); in PerformDAGCombine() local
8995 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
8997 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
8998 return DAG.getNode(ISD::BITCAST, SL, VT, Ext); in PerformDAGCombine()
9201 SDLoc SL(Node); in legalizeTargetIndependentNode() local
9208 = DAG.getCopyToReg(Node->getOperand(0), SL, VReg, SrcVal, in legalizeTargetIndependentNode()
9211 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0), in legalizeTargetIndependentNode()