Lines Matching refs:LLT
56 static LLT getPow2VectorType(LLT Ty) { in getPow2VectorType()
63 static LLT getPow2ScalarType(LLT Ty) { in getPow2ScalarType()
66 return LLT::scalar(Pow2Bits); in getPow2ScalarType()
74 const LLT Ty = Query.Types[TypeIdx]; in isSmallOddVector()
78 const LLT EltTy = Ty.getElementType(); in isSmallOddVector()
88 const LLT Ty = Query.Types[TypeIdx]; in sizeIsMultipleOf32()
95 const LLT Ty = Query.Types[TypeIdx]; in isWideVec16()
96 const LLT EltTy = Ty.getScalarType(); in isWideVec16()
103 const LLT Ty = Query.Types[TypeIdx]; in oneMoreElement()
104 const LLT EltTy = Ty.getElementType(); in oneMoreElement()
106 LLT::fixed_vector(Ty.getNumElements() + 1, EltTy)); in oneMoreElement()
112 const LLT Ty = Query.Types[TypeIdx]; in fewerEltsToSize64Vector()
113 const LLT EltTy = Ty.getElementType(); in fewerEltsToSize64Vector()
117 return std::pair(TypeIdx, LLT::scalarOrVector( in fewerEltsToSize64Vector()
126 const LLT Ty = Query.Types[TypeIdx]; in moreEltsToNext32Bit()
128 const LLT EltTy = Ty.getElementType(); in moreEltsToNext32Bit()
136 return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy)); in moreEltsToNext32Bit()
143 const LLT Ty = Query.Types[TypeIdx]; in moreElementsToNextExistingRegClass()
158 return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltSize)); in moreElementsToNextExistingRegClass()
162 static LLT getBufferRsrcScalarType(const LLT Ty) { in getBufferRsrcScalarType()
164 return LLT::scalar(128); in getBufferRsrcScalarType()
166 return LLT::vector(NumElems, LLT::scalar(128)); in getBufferRsrcScalarType()
169 static LLT getBufferRsrcRegisterType(const LLT Ty) { in getBufferRsrcRegisterType()
171 return LLT::fixed_vector(4, LLT::scalar(32)); in getBufferRsrcRegisterType()
173 return LLT::fixed_vector(NumElems * 4, LLT::scalar(32)); in getBufferRsrcRegisterType()
176 static LLT getBitcastRegisterType(const LLT Ty) { in getBitcastRegisterType()
182 return LLT::scalar(Size); in getBitcastRegisterType()
185 return LLT::scalarOrVector(ElementCount::getFixed(Size / 32), 32); in getBitcastRegisterType()
190 const LLT Ty = Query.Types[TypeIdx]; in bitcastToRegisterType()
197 const LLT Ty = Query.Types[TypeIdx]; in bitcastToVectorElement32()
201 TypeIdx, LLT::scalarOrVector(ElementCount::getFixed(Size / 32), 32)); in bitcastToVectorElement32()
207 const LLT QueryTy = Query.Types[TypeIdx]; in vectorSmallerThan()
214 const LLT QueryTy = Query.Types[TypeIdx]; in vectorWiderThan()
221 const LLT QueryTy = Query.Types[TypeIdx]; in numElementsNotEven()
230 static bool isRegisterVectorElementType(LLT EltTy) { in isRegisterVectorElementType()
235 static bool isRegisterVectorType(LLT Ty) { in isRegisterVectorType()
242 static bool isRegisterType(LLT Ty) { in isRegisterType()
263 LLT Ty = Query.Types[TypeIdx]; in isIllegalRegisterType()
271 const LLT QueryTy = Query.Types[TypeIdx]; in elementTypeIsLegal()
274 const LLT EltTy = QueryTy.getElementType(); in elementTypeIsLegal()
275 return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32; in elementTypeIsLegal()
283 const LLT Ty = Query.Types[TypeIdx]; in isWideScalarExtLoadTruncStore()
321 const LLT Ty = Query.Types[0]; in isLoadStoreSizeLegal()
392 static bool hasBufferRsrcWorkaround(const LLT Ty) { in hasBufferRsrcWorkaround()
396 const LLT ElemTy = Ty.getElementType(); in hasBufferRsrcWorkaround()
406 static bool loadStoreBitcastWorkaround(const LLT Ty) { in loadStoreBitcastWorkaround()
419 LLT EltTy = Ty.getElementType(); in loadStoreBitcastWorkaround()
428 const LLT Ty = Query.Types[0]; in isLoadStoreLegal()
435 static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty, in shouldBitcastLoadStoreType()
436 const LLT MemTy) { in shouldBitcastLoadStoreType()
454 static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy, in shouldWidenLoad()
501 static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B, in castBufferRsrcFromV4I32()
505 const LLT PointerTy = MRI.getType(MO.getReg()); in castBufferRsrcFromV4I32()
511 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy); in castBufferRsrcFromV4I32()
512 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy); in castBufferRsrcFromV4I32()
516 const LLT S32 = LLT::scalar(32); in castBufferRsrcFromV4I32()
544 const LLT PointerTy = MRI.getType(Pointer); in castBufferRsrcToV4I32()
545 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy); in castBufferRsrcToV4I32()
546 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy); in castBufferRsrcToV4I32()
552 auto Unmerged = B.buildUnmerge(LLT::scalar(32), Pointer); in castBufferRsrcToV4I32()
565 const LLT PointerTy = B.getMRI()->getType(MO.getReg()); in castBufferRsrcArgToV4I32()
578 return LLT::pointer(AS, TM.getPointerSizeInBits(AS)); in AMDGPULegalizerInfo()
581 const LLT S1 = LLT::scalar(1); in AMDGPULegalizerInfo()
582 const LLT S8 = LLT::scalar(8); in AMDGPULegalizerInfo()
583 const LLT S16 = LLT::scalar(16); in AMDGPULegalizerInfo()
584 const LLT S32 = LLT::scalar(32); in AMDGPULegalizerInfo()
585 const LLT S64 = LLT::scalar(64); in AMDGPULegalizerInfo()
586 const LLT S128 = LLT::scalar(128); in AMDGPULegalizerInfo()
587 const LLT S256 = LLT::scalar(256); in AMDGPULegalizerInfo()
588 const LLT S512 = LLT::scalar(512); in AMDGPULegalizerInfo()
589 const LLT MaxScalar = LLT::scalar(MaxRegisterSize); in AMDGPULegalizerInfo()
591 const LLT V2S8 = LLT::fixed_vector(2, 8); in AMDGPULegalizerInfo()
592 const LLT V2S16 = LLT::fixed_vector(2, 16); in AMDGPULegalizerInfo()
593 const LLT V4S16 = LLT::fixed_vector(4, 16); in AMDGPULegalizerInfo()
595 const LLT V2S32 = LLT::fixed_vector(2, 32); in AMDGPULegalizerInfo()
596 const LLT V3S32 = LLT::fixed_vector(3, 32); in AMDGPULegalizerInfo()
597 const LLT V4S32 = LLT::fixed_vector(4, 32); in AMDGPULegalizerInfo()
598 const LLT V5S32 = LLT::fixed_vector(5, 32); in AMDGPULegalizerInfo()
599 const LLT V6S32 = LLT::fixed_vector(6, 32); in AMDGPULegalizerInfo()
600 const LLT V7S32 = LLT::fixed_vector(7, 32); in AMDGPULegalizerInfo()
601 const LLT V8S32 = LLT::fixed_vector(8, 32); in AMDGPULegalizerInfo()
602 const LLT V9S32 = LLT::fixed_vector(9, 32); in AMDGPULegalizerInfo()
603 const LLT V10S32 = LLT::fixed_vector(10, 32); in AMDGPULegalizerInfo()
604 const LLT V11S32 = LLT::fixed_vector(11, 32); in AMDGPULegalizerInfo()
605 const LLT V12S32 = LLT::fixed_vector(12, 32); in AMDGPULegalizerInfo()
606 const LLT V13S32 = LLT::fixed_vector(13, 32); in AMDGPULegalizerInfo()
607 const LLT V14S32 = LLT::fixed_vector(14, 32); in AMDGPULegalizerInfo()
608 const LLT V15S32 = LLT::fixed_vector(15, 32); in AMDGPULegalizerInfo()
609 const LLT V16S32 = LLT::fixed_vector(16, 32); in AMDGPULegalizerInfo()
610 const LLT V32S32 = LLT::fixed_vector(32, 32); in AMDGPULegalizerInfo()
612 const LLT V2S64 = LLT::fixed_vector(2, 64); in AMDGPULegalizerInfo()
613 const LLT V3S64 = LLT::fixed_vector(3, 64); in AMDGPULegalizerInfo()
614 const LLT V4S64 = LLT::fixed_vector(4, 64); in AMDGPULegalizerInfo()
615 const LLT V5S64 = LLT::fixed_vector(5, 64); in AMDGPULegalizerInfo()
616 const LLT V6S64 = LLT::fixed_vector(6, 64); in AMDGPULegalizerInfo()
617 const LLT V7S64 = LLT::fixed_vector(7, 64); in AMDGPULegalizerInfo()
618 const LLT V8S64 = LLT::fixed_vector(8, 64); in AMDGPULegalizerInfo()
619 const LLT V16S64 = LLT::fixed_vector(16, 64); in AMDGPULegalizerInfo()
621 std::initializer_list<LLT> AllS32Vectors = in AMDGPULegalizerInfo()
624 std::initializer_list<LLT> AllS64Vectors = in AMDGPULegalizerInfo()
627 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS); in AMDGPULegalizerInfo()
628 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS); in AMDGPULegalizerInfo()
629 const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT); in AMDGPULegalizerInfo()
630 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS); in AMDGPULegalizerInfo()
631 const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS); in AMDGPULegalizerInfo()
632 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS); in AMDGPULegalizerInfo()
633 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS); in AMDGPULegalizerInfo()
634 const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER); in AMDGPULegalizerInfo()
635 const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE); in AMDGPULegalizerInfo()
636 const LLT BufferStridedPtr = in AMDGPULegalizerInfo()
639 const LLT CodePtr = FlatPtr; in AMDGPULegalizerInfo()
641 const std::initializer_list<LLT> AddrSpaces64 = { in AMDGPULegalizerInfo()
645 const std::initializer_list<LLT> AddrSpaces32 = { in AMDGPULegalizerInfo()
649 const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr}; in AMDGPULegalizerInfo()
651 const std::initializer_list<LLT> FPTypesBase = { in AMDGPULegalizerInfo()
655 const std::initializer_list<LLT> FPTypes16 = { in AMDGPULegalizerInfo()
659 const std::initializer_list<LLT> FPTypesPK16 = { in AMDGPULegalizerInfo()
663 const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32; in AMDGPULegalizerInfo()
1312 1, LLT::scalar(Query.Types[0].getSizeInBits())); in AMDGPULegalizerInfo()
1315 return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits())); in AMDGPULegalizerInfo()
1328 0, LLT::scalar(Query.Types[1].getSizeInBits())); in AMDGPULegalizerInfo()
1331 return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits())); in AMDGPULegalizerInfo()
1340 const LLT DstTy = Query.Types[0]; in AMDGPULegalizerInfo()
1348 const LLT PtrTy = Query.Types[1]; in AMDGPULegalizerInfo()
1462 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> { in AMDGPULegalizerInfo()
1463 const LLT DstTy = Query.Types[0]; in AMDGPULegalizerInfo()
1464 const LLT PtrTy = Query.Types[1]; in AMDGPULegalizerInfo()
1471 return std::pair(0, LLT::scalar(MemSize)); in AMDGPULegalizerInfo()
1477 return std::pair(0, LLT::scalar(MaxSize)); in AMDGPULegalizerInfo()
1480 return std::pair(0, LLT::scalar(Align)); in AMDGPULegalizerInfo()
1487 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> { in AMDGPULegalizerInfo()
1488 const LLT DstTy = Query.Types[0]; in AMDGPULegalizerInfo()
1489 const LLT PtrTy = Query.Types[1]; in AMDGPULegalizerInfo()
1491 LLT EltTy = DstTy.getElementType(); in AMDGPULegalizerInfo()
1508 0, LLT::scalarOrVector( in AMDGPULegalizerInfo()
1521 LLT::fixed_vector(NumElts / NumPieces, EltTy)); in AMDGPULegalizerInfo()
1536 0, LLT::scalarOrVector( in AMDGPULegalizerInfo()
1630 LLT::fixed_vector(2, LocalPtr), in AMDGPULegalizerInfo()
1631 LLT::fixed_vector(2, PrivatePtr)}, in AMDGPULegalizerInfo()
1660 const LLT ValTy = Query.Types[0]; in AMDGPULegalizerInfo()
1661 const LLT AmountTy = Query.Types[1]; in AMDGPULegalizerInfo()
1696 const LLT EltTy = Query.Types[EltTypeIdx]; in AMDGPULegalizerInfo()
1697 const LLT VecTy = Query.Types[VecTypeIdx]; in AMDGPULegalizerInfo()
1698 const LLT IdxTy = Query.Types[IdxTypeIdx]; in AMDGPULegalizerInfo()
1725 const LLT EltTy = Query.Types[EltTypeIdx]; in AMDGPULegalizerInfo()
1726 const LLT VecTy = Query.Types[VecTypeIdx]; in AMDGPULegalizerInfo()
1733 LLT::fixed_vector(VecSize / TargetEltSize, TargetEltSize)); in AMDGPULegalizerInfo()
1750 const LLT &EltTy = Query.Types[1].getElementType(); in AMDGPULegalizerInfo()
1765 const LLT BigTy = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1770 const LLT BigTy = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1771 const LLT LitTy = Query.Types[LitTyIdx]; in AMDGPULegalizerInfo()
1777 const LLT BigTy = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1783 const LLT LitTy = Query.Types[LitTyIdx]; in AMDGPULegalizerInfo()
1837 const LLT Ty = Query.Types[TypeIdx]; in AMDGPULegalizerInfo()
1839 const LLT &EltTy = Ty.getElementType(); in AMDGPULegalizerInfo()
1852 const LLT BigTy = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1881 const LLT Ty = Query.Types[LitTyIdx]; in AMDGPULegalizerInfo()
1889 const LLT Ty = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1895 const LLT &Ty = Query.Types[BigTyIdx]; in AMDGPULegalizerInfo()
1902 return std::pair(BigTyIdx, LLT::scalar(NewSizeInBits)); in AMDGPULegalizerInfo()
2111 const LLT S32 = LLT::scalar(32); in getSegmentAperture()
2112 const LLT S64 = LLT::scalar(64); in getSegmentAperture()
2139 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in getSegmentAperture()
2151 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in getSegmentAperture()
2161 LLT::scalar(32), commonAlignment(Align(64), Offset)); in getSegmentAperture()
2165 B.buildConstant(LLT::scalar(64), Offset).getReg(0)); in getSegmentAperture()
2171 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in getSegmentAperture()
2184 LLT::scalar(32), commonAlignment(Align(64), StructOffset)); in getSegmentAperture()
2187 B.buildConstant(LLT::scalar(64), StructOffset).getReg(0)); in getSegmentAperture()
2217 const LLT S32 = LLT::scalar(32); in legalizeAddrSpaceCast()
2221 LLT DstTy = MRI.getType(Dst); in legalizeAddrSpaceCast()
2222 LLT SrcTy = MRI.getType(Src); in legalizeAddrSpaceCast()
2257 B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0)); in legalizeAddrSpaceCast()
2287 auto CmpRes = B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, in legalizeAddrSpaceCast()
2329 LLT Ty = MRI.getType(Src); in legalizeFroundeven()
2345 auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2); in legalizeFroundeven()
2355 const LLT S1 = LLT::scalar(1); in legalizeFceil()
2356 const LLT S64 = LLT::scalar(64); in legalizeFceil()
2387 LLT Ty = MRI.getType(DstReg); in legalizeFrem()
2401 LLT S32 = LLT::scalar(32); in extractF64Exponent()
2417 const LLT S1 = LLT::scalar(1); in legalizeIntrinsicTrunc()
2418 const LLT S32 = LLT::scalar(32); in legalizeIntrinsicTrunc()
2419 const LLT S64 = LLT::scalar(64); in legalizeIntrinsicTrunc()
2466 const LLT S64 = LLT::scalar(64); in legalizeITOFP()
2467 const LLT S32 = LLT::scalar(32); in legalizeITOFP()
2524 const LLT S64 = LLT::scalar(64); in legalizeFPTOI()
2525 const LLT S32 = LLT::scalar(32); in legalizeFPTOI()
2527 const LLT SrcLT = MRI.getType(Src); in legalizeFPTOI()
2615 LLT VecTy = MRI.getType(Vec); in legalizeExtractVectorElt()
2616 LLT EltTy = VecTy.getElementType(); in legalizeExtractVectorElt()
2625 LLT IntTy = LLT::scalar(EltTy.getSizeInBits()); in legalizeExtractVectorElt()
2626 LLT IntVecTy = VecTy.changeElementType(IntTy); in legalizeExtractVectorElt()
2667 LLT VecTy = MRI.getType(Vec); in legalizeInsertVectorElt()
2668 LLT EltTy = VecTy.getElementType(); in legalizeInsertVectorElt()
2678 LLT IntTy = LLT::scalar(EltTy.getSizeInBits()); in legalizeInsertVectorElt()
2679 LLT IntVecTy = VecTy.changeElementType(IntTy); in legalizeInsertVectorElt()
2723 LLT Ty = MRI.getType(DstReg); in legalizeSinCos()
2746 bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy, in buildPCRelGlobalAddress()
2776 LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64); in buildPCRelGlobalAddress()
2800 Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, in buildAbsGlobalAddress()
2804 LLT S32 = LLT::scalar(32); in buildAbsGlobalAddress()
2836 : MRI.createGenericVirtualRegister(LLT::scalar(64)); in buildAbsGlobalAddress()
2858 LLT Ty = MRI.getType(DstReg); in legalizeGlobalValue()
2904 LLT S32 = LLT::scalar(32); in legalizeGlobalValue()
2938 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64); in legalizeGlobalValue()
2941 LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty; in legalizeGlobalValue()
2961 static LLT widenToNextPowerOf2(LLT Ty) { in widenToNextPowerOf2()
2965 return LLT::scalar(PowerOf2Ceil(Ty.getSizeInBits())); in widenToNextPowerOf2()
2975 LLT PtrTy = MRI.getType(PtrReg); in legalizeLoad()
2979 LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64); in legalizeLoad()
2991 LLT ValTy = MRI.getType(ValReg); in legalizeLoad()
3002 const LLT MemTy = MMO->getMemoryType(); in legalizeLoad()
3028 LLT WideTy = widenToNextPowerOf2(ValTy); in legalizeLoad()
3064 LLT DataTy = MRI.getType(DataReg); in legalizeStore()
3078 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in legalizeFMad()
3086 if (Ty == LLT::float32() && in legalizeFMad()
3089 if (Ty == LLT::float16() && in legalizeFMad()
3109 LLT ValTy = MRI.getType(CmpVal); in legalizeAtomicCmpXChg()
3110 LLT VecTy = LLT::fixed_vector(2, ValTy); in legalizeAtomicCmpXChg()
3145 return MRI.getType(DefMI->getOperand(1).getReg()) == LLT::scalar(16); in valueIsKnownNeverF32Denorm()
3174 const LLT F32 = LLT::scalar(32); in getScaledLogInput()
3178 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, SmallestNormal); in getScaledLogInput()
3199 LLT Ty = B.getMRI()->getType(Dst); in legalizeFlog2()
3202 if (Ty == LLT::scalar(16)) { in legalizeFlog2()
3203 const LLT F32 = LLT::scalar(32); in legalizeFlog2()
3214 assert(Ty == LLT::scalar(32)); in legalizeFlog2()
3239 static Register getMad(MachineIRBuilder &B, LLT Ty, Register X, Register Y, in getMad()
3254 const LLT Ty = MRI.getType(X); in legalizeFlogCommon()
3257 const LLT F32 = LLT::scalar(32); in legalizeFlogCommon()
3258 const LLT F16 = LLT::scalar(16); in legalizeFlogCommon()
3335 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags); in legalizeFlogCommon()
3359 LLT Ty = B.getMRI()->getType(Dst); in legalizeFlogUnsafe()
3361 if (Ty == LLT::scalar(32)) { in legalizeFlogUnsafe()
3384 auto Log2Operand = Ty == LLT::scalar(16) in legalizeFlogUnsafe()
3402 LLT Ty = B.getMRI()->getType(Dst); in legalizeFExp2()
3403 const LLT F16 = LLT::scalar(16); in legalizeFExp2()
3404 const LLT F32 = LLT::scalar(32); in legalizeFExp2()
3432 auto NeedsScaling = B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, in legalizeFExp2()
3454 LLT Ty = B.getMRI()->getType(Dst); in legalizeFExpUnsafe()
3455 LLT F32 = LLT::scalar(32); in legalizeFExpUnsafe()
3474 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold, Flags); in legalizeFExpUnsafe()
3499 LLT Ty = MRI.getType(Dst); in legalizeFExp()
3500 const LLT F16 = LLT::scalar(16); in legalizeFExp()
3501 const LLT F32 = LLT::scalar(32); in legalizeFExp()
3602 auto IntE = B.buildFPTOSI(LLT::scalar(32), E); in legalizeFExp()
3613 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, UnderflowCheckConst); in legalizeFExp()
3624 B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), X, OverflowCheckConst); in legalizeFExp()
3640 LLT Ty = B.getMRI()->getType(Dst); in legalizeFPow()
3641 const LLT F16 = LLT::float16(); in legalizeFPow()
3642 const LLT F32 = LLT::float32(); in legalizeFPow()
3684 const LLT S1 = LLT::scalar(1); in legalizeFFloor()
3685 const LLT F64 = LLT::float64(); in legalizeFFloor()
3742 const LLT S32 = LLT::scalar(32); in legalizeBuildVector()
3743 const LLT S16 = LLT::scalar(16); in legalizeBuildVector()
3744 assert(MRI.getType(Dst) == LLT::fixed_vector(2, 16)); in legalizeBuildVector()
3784 const LLT S1 = LLT::scalar(1); in buildMultiply()
3785 const LLT S32 = LLT::scalar(32); in buildMultiply()
3786 const LLT S64 = LLT::scalar(64); in buildMultiply()
4036 LLT Ty = MRI.getType(DstReg); in legalizeMul()
4054 LLT S32 = LLT::scalar(32); in legalizeMul()
4080 LLT DstTy = MRI.getType(Dst); in legalizeCTLZ_CTTZ()
4081 LLT SrcTy = MRI.getType(Src); in legalizeCTLZ_CTTZ()
4147 LLT ArgTy) const { in loadInputValue()
4156 const LLT S32 = LLT::scalar(32); in loadInputValue()
4183 LLT ArgTy; in loadInputValue()
4201 ArgTy = LLT::scalar(32); in loadInputValue()
4206 ArgTy = LLT::scalar(32); in loadInputValue()
4211 ArgTy = LLT::scalar(32); in loadInputValue()
4267 LLT ArgTy; in legalizeWorkitemIDIntrinsic()
4287 Register TmpReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); in legalizeWorkitemIDIntrinsic()
4299 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64); in getKernargParameterPtr()
4308 auto COffset = B.buildConstant(LLT::scalar(64), Offset); in getKernargParameterPtr()
4321 assert(B.getMRI()->getType(DstReg) == LLT::scalar(32) && in legalizeKernargMemParameter()
4337 LLT DstTy = MRI.getType(Dst); in legalizeFDIV()
4338 LLT S16 = LLT::scalar(16); in legalizeFDIV()
4339 LLT S32 = LLT::scalar(32); in legalizeFDIV()
4340 LLT S64 = LLT::scalar(64); in legalizeFDIV()
4357 const LLT S1 = LLT::scalar(1); in legalizeUnsignedDIV_REM32Impl()
4358 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM32Impl()
4410 const LLT S32 = LLT::scalar(32); in emitReciprocalU64()
4445 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM64Impl()
4446 const LLT S64 = LLT::scalar(64); in legalizeUnsignedDIV_REM64Impl()
4447 const LLT S1 = LLT::scalar(1); in legalizeUnsignedDIV_REM64Impl()
4574 const LLT S64 = LLT::scalar(64); in legalizeUnsignedDIV_REM()
4575 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM()
4579 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in legalizeUnsignedDIV_REM()
4595 const LLT S64 = LLT::scalar(64); in legalizeSignedDIV_REM()
4596 const LLT S32 = LLT::scalar(32); in legalizeSignedDIV_REM()
4598 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in legalizeSignedDIV_REM()
4667 LLT ResTy = MRI.getType(Res); in legalizeFastUnsafeFDIV()
4674 if (!AllowInaccurateRcp && ResTy != LLT::scalar(16)) in legalizeFastUnsafeFDIV()
4708 if (!AllowInaccurateRcp && (ResTy != LLT::scalar(16) || in legalizeFastUnsafeFDIV()
4729 LLT ResTy = MRI.getType(Res); in legalizeFastUnsafeFDIV64()
4771 LLT S16 = LLT::scalar(16); in legalizeFDIV16()
4772 LLT S32 = LLT::scalar(32); in legalizeFDIV16()
4836 LLT S32 = LLT::scalar(32); in legalizeFDIV32()
4837 LLT S1 = LLT::scalar(1); in legalizeFDIV32()
4921 LLT S64 = LLT::scalar(64); in legalizeFDIV64()
4922 LLT S1 = LLT::scalar(1); in legalizeFDIV64()
4957 LLT S32 = LLT::scalar(32); in legalizeFDIV64()
4998 LLT Ty = MRI.getType(Res0); in legalizeFFREXP()
4999 LLT InstrExpTy = Ty == LLT::scalar(16) ? LLT::scalar(16) : LLT::scalar(32); in legalizeFFREXP()
5012 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags); in legalizeFFREXP()
5033 LLT S32 = LLT::scalar(32); in legalizeFDIVFastIntrin()
5034 LLT S1 = LLT::scalar(1); in legalizeFDIVFastIntrin()
5067 const LLT F32 = LLT::scalar(32); in legalizeFSQRTF16()
5084 const LLT S1 = LLT::scalar(1); in legalizeFSQRTF32()
5085 const LLT F32 = LLT::scalar(32); in legalizeFSQRTF32()
5086 const LLT I32 = LLT::scalar(32); in legalizeFSQRTF32()
5151 auto IsZeroOrInf = B.buildIsFPClass(LLT::scalar(1), SqrtX, fcZero | fcPosInf); in legalizeFSQRTF32()
5181 const LLT S1 = LLT::scalar(1); in legalizeFSQRTF64()
5182 const LLT S32 = LLT::scalar(32); in legalizeFSQRTF64()
5183 const LLT F64 = LLT::scalar(64); in legalizeFSQRTF64()
5233 auto IsZeroOrInf = B.buildIsFPClass(LLT::scalar(1), SqrtX, fcZero | fcPosInf); in legalizeFSQRTF64()
5245 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in legalizeFSQRT()
5246 if (Ty == LLT::scalar(32)) in legalizeFSQRT()
5248 if (Ty == LLT::scalar(64)) in legalizeFSQRT()
5250 if (Ty == LLT::scalar(16)) in legalizeFSQRT()
5271 LLT Ty = MRI.getType(Dst); in legalizeRsqClampIntrinsic()
5274 if (Ty == LLT::scalar(32)) in legalizeRsqClampIntrinsic()
5276 else if (Ty == LLT::scalar(64)) in legalizeRsqClampIntrinsic()
5341 LLT DstTy = MRI.getType(DstReg); in getImplicitArgPtr()
5342 LLT IdxTy = LLT::scalar(DstTy.getSizeInBits()); in getImplicitArgPtr()
5366 LLT S32 = LLT::scalar(32); in legalizePointerAsRsrcIntrin()
5449 auto Unmerge = B.buildUnmerge(LLT::scalar(32), MI.getOperand(2).getReg()); in legalizeIsAddrSpace()
5469 const LLT S32 = LLT::scalar(32); in splitBufferOffsets()
5514 const LLT S16 = LLT::scalar(16); in handleD16VData()
5515 const LLT S32 = LLT::scalar(32); in handleD16VData()
5516 LLT StoreVT = MRI.getType(Reg); in handleD16VData()
5528 return B.buildBuildVector(LLT::fixed_vector(NumElts, S32), WideRegs) in handleD16VData()
5538 return B.buildBuildVector(LLT::fixed_vector(2, S32), PackedRegs) in handleD16VData()
5548 Reg = B.buildBuildVector(LLT::fixed_vector(6, S16), PackedRegs).getReg(0); in handleD16VData()
5549 return B.buildBitcast(LLT::fixed_vector(3, S32), Reg).getReg(0); in handleD16VData()
5554 Reg = B.buildBitcast(LLT::fixed_vector(2, S32), Reg).getReg(0); in handleD16VData()
5559 return B.buildBuildVector(LLT::fixed_vector(4, S32), PackedRegs) in handleD16VData()
5566 if (StoreVT == LLT::fixed_vector(3, S16)) { in handleD16VData()
5567 Reg = B.buildPadVectorWithUndefElements(LLT::fixed_vector(4, S16), Reg) in handleD16VData()
5576 LLT Ty = MRI->getType(VData); in fixStoreSourceType()
5578 const LLT S16 = LLT::scalar(16); in fixStoreSourceType()
5585 if (Ty == LLT::scalar(8) || Ty == S16) { in fixStoreSourceType()
5586 Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0); in fixStoreSourceType()
5606 LLT Ty = MRI.getType(VData); in legalizeBufferStore()
5607 LLT EltTy = Ty.getScalarType(); in legalizeBufferStore()
5609 const LLT S32 = LLT::scalar(32); in legalizeBufferStore()
5715 const LLT MemTy = MMO->getMemoryType(); in legalizeBufferLoad()
5716 const LLT S32 = LLT::scalar(32); in legalizeBufferLoad()
5757 LLT Ty = MRI.getType(Dst); in legalizeBufferLoad()
5764 LLT EltTy = Ty.getScalarType(); in legalizeBufferLoad()
5806 LLT LoadTy = LLT::fixed_vector(NumLoadDWords, S32); in legalizeBufferLoad()
5829 LLT UnpackedTy = Ty.changeElementSize(32); in legalizeBufferLoad()
5975 VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0); in legalizeBufferAtomic()
6014 const LLT S16 = LLT::scalar(16); in packImage16bitOpsToDwords()
6015 const LLT V2S16 = LLT::fixed_vector(2, 16); in packImage16bitOpsToDwords()
6072 const LLT S32 = LLT::scalar(32); in convertImageAddrToPacked()
6086 B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, 32), AddrRegs); in convertImageAddrToPacked()
6126 const LLT S32 = LLT::scalar(32); in legalizeImageIntrinsic()
6127 const LLT S16 = LLT::scalar(16); in legalizeImageIntrinsic()
6128 const LLT V2S16 = LLT::fixed_vector(2, 16); in legalizeImageIntrinsic()
6132 LLT Ty = MRI->getType(VData); in legalizeImageIntrinsic()
6139 LLT GradTy = in legalizeImageIntrinsic()
6141 LLT AddrTy = in legalizeImageIntrinsic()
6185 LLT Ty = MRI->getType(VData0); in legalizeImageIntrinsic()
6194 LLT PackedTy = LLT::fixed_vector(2, Ty); in legalizeImageIntrinsic()
6234 LLT PackedAddrTy = in legalizeImageIntrinsic()
6235 LLT::fixed_vector(2 * (PackedRegs.size() - NSAMaxSize + 1), 16); in legalizeImageIntrinsic()
6241 LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), 16); in legalizeImageIntrinsic()
6314 const LLT EltTy = Ty.getScalarType(); in legalizeImageIntrinsic()
6332 const LLT AdjustedTy = in legalizeImageIntrinsic()
6340 LLT RoundedTy; in legalizeImageIntrinsic()
6343 LLT TFETy; in legalizeImageIntrinsic()
6346 LLT RegTy; in legalizeImageIntrinsic()
6350 LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), 32); in legalizeImageIntrinsic()
6351 TFETy = LLT::fixed_vector(AdjustedNumElts + 1, 32); in legalizeImageIntrinsic()
6357 RoundedTy = LLT::scalarOrVector( in legalizeImageIntrinsic()
6359 TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, S32); in legalizeImageIntrinsic()
6375 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy; in legalizeImageIntrinsic()
6453 auto padWithUndef = [&](LLT Ty, int NumElts) { in legalizeImageIntrinsic()
6462 LLT ResTy = MRI->getType(ResultRegs[0]); in legalizeImageIntrinsic()
6473 const LLT V3S16 = LLT::fixed_vector(3, 16); in legalizeImageIntrinsic()
6479 LLT V4S16 = LLT::fixed_vector(4, 16); in legalizeImageIntrinsic()
6507 LLT Ty = B.getMRI()->getType(OrigDst); in legalizeSBufferLoad()
6517 Dst = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32)); in legalizeSBufferLoad()
6530 if (shouldBitcastLoadStoreType(ST, Ty, LLT::scalar(Size))) { in legalizeSBufferLoad()
6616 const LLT S64 = LLT::scalar(64); in legalizeTrapHsaQueuePtr()
6628 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in legalizeTrapHsaQueuePtr()
6640 LLT::scalar(64), commonAlignment(Align(64), Offset)); in legalizeTrapHsaQueuePtr()
6644 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in legalizeTrapHsaQueuePtr()
6646 B.buildConstant(LLT::scalar(64), Offset).getReg(0)); in legalizeTrapHsaQueuePtr()
6660 MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); in legalizeTrapHsaQueuePtr()
6705 const LLT S16 = LLT::scalar(16); in legalizeBVHIntrinsic()
6706 const LLT S32 = LLT::scalar(32); in legalizeBVHIntrinsic()
6707 const LLT V2S16 = LLT::fixed_vector(2, 16); in legalizeBVHIntrinsic()
6708 const LLT V3S32 = LLT::fixed_vector(3, 32); in legalizeBVHIntrinsic()
6833 LLT OpTy = LLT::fixed_vector(Ops.size(), 32); in legalizeBVHIntrinsic()
6891 LLT S32 = LLT::scalar(32); in legalizeWaveID()
7198 LLT S32 = LLT::scalar(32); in legalizeIntrinsic()
7207 LLT S32 = LLT::scalar(32); in legalizeIntrinsic()