Lines Matching refs:SL
1903 const SDLoc &SL, in lowerKernArgParameterPtr() argument
1921 return DAG.getConstant(Offset, SL, PtrVT); in lowerKernArgParameterPtr()
1924 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL, in lowerKernArgParameterPtr()
1927 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset)); in lowerKernArgParameterPtr()
1931 const SDLoc &SL) const { in getImplicitArgPtr()
1934 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset); in getImplicitArgPtr()
1938 const SDLoc &SL) const { in getLDSKernelId()
1944 return DAG.getConstant(*KnownSize, SL, MVT::i32); in getLDSKernelId()
1949 const SDLoc &SL, SDValue Val, in convertArgType() argument
1958 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val, in convertArgType()
1959 DAG.getConstant(0, SL, MVT::i32)); in convertArgType()
1966 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT)); in convertArgType()
1970 Val = getFPExtOrFPRound(DAG, Val, SL, VT); in convertArgType()
1972 Val = DAG.getSExtOrTrunc(Val, SL, VT); in convertArgType()
1974 Val = DAG.getZExtOrTrunc(Val, SL, VT); in convertArgType()
1980 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain, in lowerKernargMemParameter() argument
1997 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset); in lowerKernargMemParameter()
1998 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), in lowerKernargMemParameter()
2002 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
2003 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
2005 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract); in lowerKernargMemParameter()
2006 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal); in lowerKernargMemParameter()
2007 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg); in lowerKernargMemParameter()
2010 return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL); in lowerKernargMemParameter()
2013 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset); in lowerKernargMemParameter()
2014 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment, in lowerKernargMemParameter()
2018 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg); in lowerKernargMemParameter()
2019 return DAG.getMergeValues({ Val, Load.getValue(1) }, SL); in lowerKernargMemParameter()
2023 const SDLoc &SL, SDValue Chain, in lowerStackParameter() argument
2065 ExtType, SL, VA.getLocVT(), Chain, FIN, in lowerStackParameter()
3357 SDLoc SL; in passSpecialInputs() local
3376 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, in passSpecialInputs()
3377 DAG.getShiftAmountConstant(10, MVT::i32, SL)); in passSpecialInputs()
3379 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; in passSpecialInputs()
3385 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, in passSpecialInputs()
3386 DAG.getShiftAmountConstant(20, MVT::i32, SL)); in passSpecialInputs()
3388 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; in passSpecialInputs()
3931 SDLoc SL(Op); in LowerSTACKSAVE() local
3933 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32); in LowerSTACKSAVE()
3939 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP); in LowerSTACKSAVE()
3940 return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL); in LowerSTACKSAVE()
3945 SDLoc SL(Op); in lowerGET_ROUNDING() local
3950 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerGET_ROUNDING()
3953 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_ROUNDING()
3954 SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(), in lowerGET_ROUNDING()
3986 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64); in lowerGET_ROUNDING()
3988 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerGET_ROUNDING()
3990 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two); in lowerGET_ROUNDING()
3995 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerGET_ROUNDING()
3996 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerGET_ROUNDING()
3998 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32); in lowerGET_ROUNDING()
4000 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask); in lowerGET_ROUNDING()
4004 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerGET_ROUNDING()
4006 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT); in lowerGET_ROUNDING()
4007 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four); in lowerGET_ROUNDING()
4008 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue, in lowerGET_ROUNDING()
4011 return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL); in lowerGET_ROUNDING()
4040 SDLoc SL(Op); in lowerFP_EXTEND() local
4042 DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src); in lowerFP_EXTEND()
4048 return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast); in lowerFP_EXTEND()
5454 SDLoc SL(Op); in splitUnaryVectorOp() local
5455 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, in splitUnaryVectorOp()
5457 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, in splitUnaryVectorOp()
5479 SDLoc SL(Op); in splitBinaryVectorOp() local
5481 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, in splitBinaryVectorOp()
5483 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, in splitBinaryVectorOp()
5510 SDLoc SL(Op); in splitTernaryVectorOp() local
5513 SDValue OpLo = DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2, in splitTernaryVectorOp()
5515 SDValue OpHi = DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2, in splitTernaryVectorOp()
5824 SDLoc SL(N); in lowerFCMPIntrinsic() local
5827 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
5828 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
5835 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, in lowerFCMPIntrinsic()
5839 return DAG.getZExtOrTrunc(SetCC, SL, VT); in lowerFCMPIntrinsic()
5846 SDLoc SL(N); in lowerBALLOTIntrinsic() local
5850 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0), in lowerBALLOTIntrinsic()
5856 return DAG.getConstant(0, SL, VT); in lowerBALLOTIntrinsic()
5868 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT); in lowerBALLOTIntrinsic()
5875 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), in lowerBALLOTIntrinsic()
5876 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); in lowerBALLOTIntrinsic()
5902 SDLoc SL(N); in ReplaceNodeResults() local
5903 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
5905 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
5914 SDLoc SL(N); in ReplaceNodeResults() local
5928 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1)); in ReplaceNodeResults()
5930 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
5931 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
6004 SDLoc SL(N); in ReplaceNodeResults() local
6007 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1)); in ReplaceNodeResults()
6008 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2)); in ReplaceNodeResults()
6012 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
6013 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
6017 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT, in ReplaceNodeResults()
6021 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect); in ReplaceNodeResults()
6022 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect)); in ReplaceNodeResults()
6029 SDLoc SL(N); in ReplaceNodeResults() local
6030 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6032 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
6034 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
6035 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6042 SDLoc SL(N); in ReplaceNodeResults() local
6043 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6045 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
6047 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
6048 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6381 SDLoc SL(Op); in lowerMUL() local
6384 DAG.getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0); in lowerMUL()
6389 DAG.getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0); in lowerMUL()
6396 SDLoc SL(Op); in lowerXMULO() local
6407 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); in lowerXMULO()
6408 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt); in lowerXMULO()
6409 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, in lowerXMULO()
6411 SL, VT, Result, ShiftAmt), in lowerXMULO()
6413 return DAG.getMergeValues({ Result, Overflow }, SL); in lowerXMULO()
6417 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS); in lowerXMULO()
6419 SL, VT, LHS, RHS); in lowerXMULO()
6422 ? DAG.getNode(ISD::SRA, SL, VT, Result, in lowerXMULO()
6423 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) in lowerXMULO()
6424 : DAG.getConstant(0, SL, VT); in lowerXMULO()
6425 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); in lowerXMULO()
6427 return DAG.getMergeValues({ Result, Overflow }, SL); in lowerXMULO()
6456 SDLoc SL(Op); in lowerTrapEndpgm() local
6458 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain); in lowerTrapEndpgm()
6474 SDLoc SL(Op); in lowerTrapHsaQueuePtr() local
6482 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR); in lowerTrapHsaQueuePtr()
6492 QueuePtr = DAG.getConstant(0, SL, MVT::i64); in lowerTrapHsaQueuePtr()
6500 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, in lowerTrapHsaQueuePtr()
6506 DAG.getTargetConstant(TrapID, SL, MVT::i16), in lowerTrapHsaQueuePtr()
6510 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsaQueuePtr()
6515 SDLoc SL(Op); in lowerTrapHsa() local
6521 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerTrapHsa()
6523 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsa()
6527 SDLoc SL(Op); in lowerDEBUGTRAP() local
6545 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerDEBUGTRAP()
6547 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
6637 SDLoc SL(Op); in lowerADDRSPACECAST() local
6641 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
6653 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
6659 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
6660 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
6662 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr, in lowerADDRSPACECAST()
6672 SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG); in lowerADDRSPACECAST()
6674 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
6675 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerADDRSPACECAST()
6681 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
6684 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
6686 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr, in lowerADDRSPACECAST()
6695 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32); in lowerADDRSPACECAST()
6696 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi); in lowerADDRSPACECAST()
6697 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in lowerADDRSPACECAST()
6702 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
6708 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); in lowerADDRSPACECAST()
6729 SDLoc SL(Op); in lowerINSERT_SUBVECTOR() local
6742 Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec); in lowerINSERT_SUBVECTOR()
6743 Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins); in lowerINSERT_SUBVECTOR()
6750 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins, in lowerINSERT_SUBVECTOR()
6751 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6753 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt, in lowerINSERT_SUBVECTOR()
6754 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6757 return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec); in lowerINSERT_SUBVECTOR()
6761 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins, in lowerINSERT_SUBVECTOR()
6762 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6763 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt, in lowerINSERT_SUBVECTOR()
6764 DAG.getConstant(IdxVal + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6778 SDLoc SL(Op); in lowerINSERT_VECTOR_ELT() local
6784 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
6786 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
6787 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6788 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
6789 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6791 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
6792 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
6796 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
6798 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
6799 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6801 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
6804 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
6805 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
6807 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat); in lowerINSERT_VECTOR_ELT()
6825 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
6826 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
6827 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT, in lowerINSERT_VECTOR_ELT()
6828 DAG.getConstant(EltMask, SL, IntVT), ScaledIdx); in lowerINSERT_VECTOR_ELT()
6831 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT, in lowerINSERT_VECTOR_ELT()
6832 DAG.getSplatBuildVector(VecVT, SL, InsVal)); in lowerINSERT_VECTOR_ELT()
6835 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal); in lowerINSERT_VECTOR_ELT()
6838 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerINSERT_VECTOR_ELT()
6839 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT, in lowerINSERT_VECTOR_ELT()
6840 DAG.getNOT(SL, BFM, IntVT), BCVec); in lowerINSERT_VECTOR_ELT()
6843 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS); in lowerINSERT_VECTOR_ELT()
6845 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI); in lowerINSERT_VECTOR_ELT()
6850 SDLoc SL(Op); in lowerEXTRACT_VECTOR_ELT() local
6876 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6877 DAG.getConstant(0, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
6879 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6880 DAG.getConstant(1, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
6885 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6886 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
6889 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
6891 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
6899 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6900 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
6904 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
6907 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
6914 SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT); in lowerEXTRACT_VECTOR_ELT()
6915 SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask); in lowerEXTRACT_VECTOR_ELT()
6916 SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT); in lowerEXTRACT_VECTOR_ELT()
6917 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx); in lowerEXTRACT_VECTOR_ELT()
6929 Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT); in lowerEXTRACT_VECTOR_ELT()
6935 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
6938 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
6940 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerEXTRACT_VECTOR_ELT()
6941 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx); in lowerEXTRACT_VECTOR_ELT()
6944 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
6945 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result); in lowerEXTRACT_VECTOR_ELT()
6948 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT); in lowerEXTRACT_VECTOR_ELT()
6958 SDLoc SL(Op); in lowerVECTOR_SHUFFLE() local
6982 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, in lowerVECTOR_SHUFFLE()
6984 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
6995 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in lowerVECTOR_SHUFFLE()
6996 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
6999 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in lowerVECTOR_SHUFFLE()
7000 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7001 Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 })); in lowerVECTOR_SHUFFLE()
7005 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces); in lowerVECTOR_SHUFFLE()
7014 SDLoc SL(Op); in lowerSCALAR_TO_VECTOR() local
7021 return DAG.getBuildVector(ResultVT, SL, VElts); in lowerSCALAR_TO_VECTOR()
7026 SDLoc SL(Op); in lowerBUILD_VECTOR() local
7042 SDValue Lo = DAG.getBuildVector(HalfVT, SL, LoOps); in lowerBUILD_VECTOR()
7043 SDValue Hi = DAG.getBuildVector(HalfVT, SL, HiOps); in lowerBUILD_VECTOR()
7045 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Lo); in lowerBUILD_VECTOR()
7046 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Hi); in lowerBUILD_VECTOR()
7048 SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL, in lowerBUILD_VECTOR()
7050 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7065 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]); in lowerBUILD_VECTOR()
7066 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec); in lowerBUILD_VECTOR()
7070 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts); in lowerBUILD_VECTOR()
7071 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7086 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]); in lowerBUILD_VECTOR()
7087 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec); in lowerBUILD_VECTOR()
7091 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts); in lowerBUILD_VECTOR()
7092 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7103 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7104 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7105 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo); in lowerBUILD_VECTOR()
7108 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
7109 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
7111 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
7112 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
7114 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi); in lowerBUILD_VECTOR()
7116 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7117 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7119 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
7120 return DAG.getNode(ISD::BITCAST, SL, VT, Or); in lowerBUILD_VECTOR()
7262 SDLoc SL(Op); in lowerImplicitZextParam() local
7264 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); in lowerImplicitZextParam()
7266 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
7927 SDLoc SL(Op); in lowerWaveID() local
7929 SDValue TTMP8 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::TTMP8, VT); in lowerWaveID()
7930 return DAG.getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8, in lowerWaveID()
7931 DAG.getConstant(25, SL, VT), DAG.getConstant(5, SL, VT)); in lowerWaveID()
7937 SDLoc SL(Op); in lowerWorkitemID() local
7941 return DAG.getConstant(0, SL, MVT::i32); in lowerWorkitemID()
7955 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val, in lowerWorkitemID()
8264 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8267 SDValue Aperture = getSegmentAperture(AS, SL, DAG); in LowerINTRINSIC_WO_CHAIN()
8271 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, in LowerINTRINSIC_WO_CHAIN()
8272 DAG.getConstant(1, SL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
8273 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); in LowerINTRINSIC_WO_CHAIN()
8299 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8300 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8301 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8311 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8312 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8313 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
9965 const SDLoc &SL, EVT VT) { in getLoadExtOrTrunc() argument
9967 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op); in getLoadExtOrTrunc()
9971 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
9973 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
9975 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
10005 SDLoc SL(Ld); in widenLoad() local
10013 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
10027 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
10031 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT); in widenLoad()
10043 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT); in widenLoad()
10047 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt); in widenLoad()
10049 return DAG.getMergeValues({ Cvt, NewLoad.getValue(1) }, SL); in widenLoad()
10257 SDLoc SL(Op); in lowerFastUnsafeFDIV() local
10286 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
10292 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in lowerFastUnsafeFDIV()
10293 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS); in lowerFastUnsafeFDIV()
10304 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
10305 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags); in lowerFastUnsafeFDIV()
10310 SDLoc SL(Op); in lowerFastUnsafeFDIV64() local
10321 SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y); in lowerFastUnsafeFDIV64()
10322 SDValue One = DAG.getConstantFP(1.0, SL, VT); in lowerFastUnsafeFDIV64()
10324 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y); in lowerFastUnsafeFDIV64()
10325 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); in lowerFastUnsafeFDIV64()
10327 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R); in lowerFastUnsafeFDIV64()
10328 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); in lowerFastUnsafeFDIV64()
10329 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R); in lowerFastUnsafeFDIV64()
10330 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R); in lowerFastUnsafeFDIV64()
10331 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X); in lowerFastUnsafeFDIV64()
10332 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret); in lowerFastUnsafeFDIV64()
10335 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPBinOp() argument
10339 return DAG.getNode(Opcode, SL, VT, A, B, Flags); in getFPBinOp()
10352 return DAG.getNode(Opcode, SL, VTList, in getFPBinOp()
10357 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPTernOp() argument
10361 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags); in getFPTernOp()
10374 return DAG.getNode(Opcode, SL, VTList, in getFPTernOp()
10383 SDLoc SL(Op); in LowerFDIV16() local
10387 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
10388 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
10390 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
10391 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
10393 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
10394 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
10396 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
10402 SDLoc SL(Op); in lowerFDIV_FAST() local
10406 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags); in lowerFDIV_FAST()
10409 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
10412 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
10414 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
10419 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT); in lowerFDIV_FAST()
10421 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags); in lowerFDIV_FAST()
10423 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags); in lowerFDIV_FAST()
10426 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags); in lowerFDIV_FAST()
10428 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags); in lowerFDIV_FAST()
10430 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags); in lowerFDIV_FAST()
10455 SDLoc SL(Op); in LowerFDIV32() local
10459 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
10463 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
10465 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
10469 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
10471 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
10477 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); in LowerFDIV32()
10499 SDNode *GetReg = DAG.getMachineNode(AMDGPU::S_GETREG_B32, SL, in LowerFDIV32()
10505 {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, SL); in LowerFDIV32()
10513 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue, in LowerFDIV32()
10518 SL, MVT::i32); in LowerFDIV32()
10519 EnableDenorm = DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs, in LowerFDIV32()
10529 NegDivScale0 = DAG.getMergeValues(Ops, SL); in LowerFDIV32()
10532 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
10535 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
10538 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
10541 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
10544 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, in LowerFDIV32()
10547 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
10556 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, in LowerFDIV32()
10564 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
10567 AMDGPU::S_SETREG_B32, SL, MVT::Other, in LowerFDIV32()
10571 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
10577 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
10580 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); in LowerFDIV32()
10587 SDLoc SL(Op); in LowerFDIV64() local
10591 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
10595 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X); in LowerFDIV64()
10597 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
10599 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
10601 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
10603 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
10605 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
10607 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X); in LowerFDIV64()
10609 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
10610 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
10612 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
10621 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
10624 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
10625 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
10626 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
10627 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
10629 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
10630 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
10633 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
10635 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
10637 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
10638 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
10639 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
10644 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
10647 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
10775 SDLoc SL(Op); in lowerFSQRTF16() local
10779 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags); in lowerFSQRTF16()
10781 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32); in lowerFSQRTF16()
10783 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags); in lowerFSQRTF16()
10785 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt, in lowerFSQRTF16()
10786 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFSQRTF16()
11151 SDLoc SL(N); in performSHLPtrCombine() local
11154 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1); in performSHLPtrCombine()
11155 SDValue COffset = DAG.getConstant(Offset, SL, VT); in performSHLPtrCombine()
11162 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags); in performSHLPtrCombine()
11182 SDLoc SL(N); in performMemSDNodeCombine() local
11215 const SDLoc &SL, in splitBinaryBitConstantOp() argument
11229 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi); in splitBinaryBitConstantOp()
11347 SDLoc SL(N); in performAndCombine() local
11348 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
11350 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
11351 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
11353 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE, in performAndCombine()
12138 SDLoc SL(N); in performOrCombine() local
12141 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
12146 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
12148 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
12619 SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const { in getCanonicalConstantFP() argument
12626 APFloat::getZero(C.getSemantics(), C.isNegative()), SL, VT); in getCanonicalConstantFP()
12638 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
12646 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
12650 return DAG.getConstantFP(C, SL, VT); in getCanonicalConstantFP()
12684 SDLoc SL(N); in performFCanonicalizeCombine() local
12694 NewElts[I] = getCanonicalConstantFP(DAG, SL, EltVT, in performFCanonicalizeCombine()
12700 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op); in performFCanonicalizeCombine()
12710 NewElts[1]: DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
12715 NewElts[0] : DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
12718 return DAG.getBuildVector(VT, SL, NewElts); in performFCanonicalizeCombine()
12732 SDLoc SL(N); in performFCanonicalizeCombine() local
12733 SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT, in performFCanonicalizeCombine()
12735 SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF()); in performFCanonicalizeCombine()
12738 return DAG.getNode(N0.getOpcode(), SL, VT, Canon0, Canon1); in performFCanonicalizeCombine()
12771 const SDLoc &SL, SDValue Src, in performIntMed3ImmCombine() argument
12799 return DAG.getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal); in performIntMed3ImmCombine()
12821 const SDLoc &SL, in performFPMed3ImmCombine() argument
12846 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0)); in performFPMed3ImmCombine()
12865 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), in performFPMed3ImmCombine()
12975 SDLoc SL(N); in performFMed3Combine() local
12985 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2); in performFMed3Combine()
13006 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0); in performFMed3Combine()
13088 SDLoc SL(N); in performExtractVectorEltCombine() local
13091 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx); in performExtractVectorEltCombine()
13092 return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt); in performExtractVectorEltCombine()
13101 SDLoc SL(N); in performExtractVectorEltCombine() local
13123 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, in performExtractVectorEltCombine()
13125 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, in performExtractVectorEltCombine()
13130 return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags()); in performExtractVectorEltCombine()
13137 SDLoc SL(N); in performExtractVectorEltCombine() local
13141 SDValue IC = DAG.getVectorIdxConstant(I, SL); in performExtractVectorEltCombine()
13142 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC); in performExtractVectorEltCombine()
13146 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ); in performExtractVectorEltCombine()
13165 SDLoc SL(N); in performExtractVectorEltCombine() local
13167 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec); in performExtractVectorEltCombine()
13170 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
13171 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13173 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
13174 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13178 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl); in performExtractVectorEltCombine()
13182 return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc); in performExtractVectorEltCombine()
13186 return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT); in performExtractVectorEltCombine()
13206 SDLoc SL(N); in performInsertVectorEltCombine() local
13212 SDValue IC = DAG.getConstant(I, SL, IdxVT); in performInsertVectorEltCombine()
13213 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); in performInsertVectorEltCombine()
13214 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ); in performInsertVectorEltCombine()
13218 return DAG.getBuildVector(VecVT, SL, Ops); in performInsertVectorEltCombine()
13255 SDLoc SL(N); in performFPRoundCombine() local
13279 SDValue A1 = DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, A, B); in performFPRoundCombine()
13280 SDValue B1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A, B); in performFPRoundCombine()
13281 SDValue C1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A1, C); in performFPRoundCombine()
13282 return DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, B1, C1); in performFPRoundCombine()
13342 SDLoc SL(N); in reassociateScalarOps() local
13343 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1); in reassociateScalarOps()
13344 return DAG.getNode(Opc, SL, VT, Add1, Op2); in reassociateScalarOps()
13347 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, in getMad64_32() argument
13353 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2); in getMad64_32()
13354 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad); in getMad64_32()
13371 SDLoc SL(N); in tryFoldToMad64_32() local
13432 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS); in tryFoldToMad64_32()
13433 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS); in tryFoldToMad64_32()
13434 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS); in tryFoldToMad64_32()
13449 SDValue One = DAG.getConstant(1, SL, MVT::i32); in tryFoldToMad64_32()
13451 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS); in tryFoldToMad64_32()
13452 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS); in tryFoldToMad64_32()
13454 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo); in tryFoldToMad64_32()
13458 std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32); in tryFoldToMad64_32()
13462 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One); in tryFoldToMad64_32()
13463 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo); in tryFoldToMad64_32()
13464 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13469 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One); in tryFoldToMad64_32()
13470 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi); in tryFoldToMad64_32()
13471 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13474 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi}); in tryFoldToMad64_32()
13479 Accum = DAG.getNode(ISD::TRUNCATE, SL, VT, Accum); in tryFoldToMad64_32()
13586 resolveSources(SelectionDAG &DAG, SDLoc SL, in resolveSources() argument
13593 auto EltVal = DAG.getBitcastedAnyExtOrTrunc(Elt->first, SL, MVT::i32); in resolveSources()
13599 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltVal, EltVal, in resolveSources()
13600 DAG.getConstant(Elt->second, SL, MVT::i32)); in resolveSources()
13622 DAG.getBitcastedAnyExtOrTrunc(FirstElt->first, SL, MVT::i32); in resolveSources()
13624 DAG.getBitcastedAnyExtOrTrunc(SecondElt->first, SL, MVT::i32); in resolveSources()
13626 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal, in resolveSources()
13628 DAG.getConstant(PermMask, SL, MVT::i32))); in resolveSources()
13639 DAG.getBitcastedAnyExtOrTrunc(FirstElt->first, SL, MVT::i32); in resolveSources()
13642 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltVal, EltVal, in resolveSources()
13643 DAG.getConstant(FirstElt->second, SL, MVT::i32))); in resolveSources()
13650 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1]) in resolveSources()
13737 SDLoc SL(N); in performAddCombine() local
13807 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32)); in performAddCombine()
13867 Src0 = resolveSources(DAG, SL, Src0s, false, true); in performAddCombine()
13868 Src1 = resolveSources(DAG, SL, Src1s, false, true); in performAddCombine()
13873 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32); in performAddCombine()
13877 SL, MVT::i64); in performAddCombine()
13880 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0, in performAddCombine()
13881 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1)); in performAddCombine()
13883 return DAG.getExtOrTrunc(*IsSigned, Dot, SL, VT); in performAddCombine()
13908 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
13910 return DAG.getNode(Opc, SL, VTList, Args); in performAddCombine()
13931 SDLoc SL(N); in performSubCombine() local
13949 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performSubCombine()
13951 return DAG.getNode(Opc, SL, VTList, Args); in performSubCombine()
13997 SDLoc SL(N); in performFAddCombine() local
14010 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
14011 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS); in performFAddCombine()
14022 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
14023 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS); in performFAddCombine()
14037 SDLoc SL(N); in performFSubCombine() local
14054 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFSubCombine()
14055 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFSubCombine()
14057 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS); in performFSubCombine()
14069 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT); in performFSubCombine()
14070 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS); in performFSubCombine()
14081 SDLoc SL(N); in performFDivCombine() local
14104 DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0), Flags); in performFDivCombine()
14105 return IsNegative ? DAG.getNode(ISD::FNEG, SL, VT, Rsq, Flags) : Rsq; in performFDivCombine()
14117 SDLoc SL(N); in performFMACombine() local
14181 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
14182 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
14191 SDLoc SL(N); in performSetCCCombine() local
14218 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14219 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14244 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14245 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14276 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14277 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
14287 SDLoc SL(N); in performCvtF32UByteNCombine() local
14314 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL, in performCvtF32UByteNCombine()
14333 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); in performCvtF32UByteNCombine()
14453 SDLoc SL(N); in PerformDAGCombine() local
14457 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
14459 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
14460 return DAG.getNode(ISD::BITCAST, SL, VT, Ext); in PerformDAGCombine()
14690 SDLoc SL(Node); in legalizeTargetIndependentNode() local
14697 = DAG.getCopyToReg(Node->getOperand(0), SL, VReg, SrcVal, in legalizeTargetIndependentNode()
14700 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0), in legalizeTargetIndependentNode()