Lines Matching refs:VT
164 auto addRegClassForRVV = [this](MVT VT) { in RISCVTargetLowering() argument
168 if (VT.getVectorMinNumElements() < MinElts) in RISCVTargetLowering()
171 unsigned Size = VT.getSizeInBits().getKnownMinValue(); in RISCVTargetLowering()
184 addRegisterClass(VT, RC); in RISCVTargetLowering()
187 for (MVT VT : BoolVecVTs) in RISCVTargetLowering() local
188 addRegClassForRVV(VT); in RISCVTargetLowering()
189 for (MVT VT : IntVecVTs) { in RISCVTargetLowering() local
190 if (VT.getVectorElementType() == MVT::i64 && in RISCVTargetLowering()
193 addRegClassForRVV(VT); in RISCVTargetLowering()
197 for (MVT VT : F16VecVTs) in RISCVTargetLowering() local
198 addRegClassForRVV(VT); in RISCVTargetLowering()
201 for (MVT VT : BF16VecVTs) in RISCVTargetLowering() local
202 addRegClassForRVV(VT); in RISCVTargetLowering()
205 for (MVT VT : F32VecVTs) in RISCVTargetLowering() local
206 addRegClassForRVV(VT); in RISCVTargetLowering()
209 for (MVT VT : F64VecVTs) in RISCVTargetLowering() local
210 addRegClassForRVV(VT); in RISCVTargetLowering()
213 auto addRegClassForFixedVectors = [this](MVT VT) { in RISCVTargetLowering() argument
214 MVT ContainerVT = getContainerForFixedLengthVector(VT); in RISCVTargetLowering()
217 addRegisterClass(VT, TRI.getRegClass(RCID)); in RISCVTargetLowering()
219 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) in RISCVTargetLowering() local
220 if (useRVVForFixedLengthVectorVT(VT)) in RISCVTargetLowering()
221 addRegClassForFixedVectors(VT); in RISCVTargetLowering()
223 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) in RISCVTargetLowering() local
224 if (useRVVForFixedLengthVectorVT(VT)) in RISCVTargetLowering()
225 addRegClassForFixedVectors(VT); in RISCVTargetLowering()
717 for (MVT VT : BoolVecVTs) { in RISCVTargetLowering() local
718 if (!isTypeLegal(VT)) in RISCVTargetLowering()
721 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
727 VT, Custom); in RISCVTargetLowering()
729 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
732 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
734 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT, in RISCVTargetLowering()
737 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom); in RISCVTargetLowering()
740 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
744 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
755 VT, Custom); in RISCVTargetLowering()
756 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
762 setTruncStoreAction(VT, OtherVT, Expand); in RISCVTargetLowering()
763 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
769 VT, Custom); in RISCVTargetLowering()
771 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
772 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
774 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom); in RISCVTargetLowering()
776 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); in RISCVTargetLowering()
777 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); in RISCVTargetLowering()
780 ISD::VECTOR_SPLICE, VT, in RISCVTargetLowering()
781 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount())); in RISCVTargetLowering()
784 for (MVT VT : IntVecVTs) { in RISCVTargetLowering() local
785 if (!isTypeLegal(VT)) in RISCVTargetLowering()
788 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
789 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
792 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); in RISCVTargetLowering()
795 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) in RISCVTargetLowering()
796 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand); in RISCVTargetLowering()
798 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT, in RISCVTargetLowering()
803 VT, Custom); in RISCVTargetLowering()
813 VT, Custom); in RISCVTargetLowering()
814 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
816 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom); in RISCVTargetLowering()
819 VT, Legal); in RISCVTargetLowering()
823 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
826 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
831 setOperationAction(IntegerVecReduceOps, VT, Custom); in RISCVTargetLowering()
833 setOperationAction(IntegerVPOps, VT, Custom); in RISCVTargetLowering()
835 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
838 VT, Custom); in RISCVTargetLowering()
843 VT, Custom); in RISCVTargetLowering()
847 VT, Custom); in RISCVTargetLowering()
849 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
850 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
852 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom); in RISCVTargetLowering()
855 setTruncStoreAction(VT, OtherVT, Expand); in RISCVTargetLowering()
856 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
860 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
861 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
864 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in RISCVTargetLowering()
867 setOperationAction(ISD::BSWAP, VT, Legal); in RISCVTargetLowering()
868 setOperationAction(ISD::VP_BSWAP, VT, Custom); in RISCVTargetLowering()
870 setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand); in RISCVTargetLowering()
871 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand); in RISCVTargetLowering()
875 setOperationAction(ISD::BITREVERSE, VT, Legal); in RISCVTargetLowering()
876 setOperationAction(ISD::VP_BITREVERSE, VT, Custom); in RISCVTargetLowering()
879 VT, Custom); in RISCVTargetLowering()
881 setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand); in RISCVTargetLowering()
882 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand); in RISCVTargetLowering()
885 VT, Expand); in RISCVTargetLowering()
889 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in RISCVTargetLowering()
894 VT, Custom); in RISCVTargetLowering()
935 const auto SetCommonVFPActions = [&](MVT VT) { in RISCVTargetLowering() argument
936 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
941 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
943 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
946 setCondCodeAction(VFPCCToExpand, VT, Expand); in RISCVTargetLowering()
948 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal); in RISCVTargetLowering()
949 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom); in RISCVTargetLowering()
954 VT, Custom); in RISCVTargetLowering()
956 setOperationAction(FloatingPointVecReduceOps, VT, Custom); in RISCVTargetLowering()
959 setOperationAction(ISD::FREM, VT, Expand); in RISCVTargetLowering()
960 setOperationAction(ISD::FPOW, VT, Expand); in RISCVTargetLowering()
961 setOperationAction(ISD::FCOS, VT, Expand); in RISCVTargetLowering()
962 setOperationAction(ISD::FSIN, VT, Expand); in RISCVTargetLowering()
963 setOperationAction(ISD::FSINCOS, VT, Expand); in RISCVTargetLowering()
964 setOperationAction(ISD::FEXP, VT, Expand); in RISCVTargetLowering()
965 setOperationAction(ISD::FEXP2, VT, Expand); in RISCVTargetLowering()
966 setOperationAction(ISD::FEXP10, VT, Expand); in RISCVTargetLowering()
967 setOperationAction(ISD::FLOG, VT, Expand); in RISCVTargetLowering()
968 setOperationAction(ISD::FLOG2, VT, Expand); in RISCVTargetLowering()
969 setOperationAction(ISD::FLOG10, VT, Expand); in RISCVTargetLowering()
971 setOperationAction(ISD::FCOPYSIGN, VT, Legal); in RISCVTargetLowering()
973 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
976 VT, Custom); in RISCVTargetLowering()
981 VT, Custom); in RISCVTargetLowering()
983 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
984 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
988 VT, Custom); in RISCVTargetLowering()
990 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
991 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
993 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom); in RISCVTargetLowering()
995 setOperationAction(FloatingPointVPOps, VT, Custom); in RISCVTargetLowering()
997 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT, in RISCVTargetLowering()
1001 VT, Legal); in RISCVTargetLowering()
1006 VT, Custom); in RISCVTargetLowering()
1012 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) { in RISCVTargetLowering() argument
1014 setTruncStoreAction(VT, SmallVT, Expand); in RISCVTargetLowering()
1015 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand); in RISCVTargetLowering()
1020 for (MVT VT : F16VecVTs) { in RISCVTargetLowering() local
1021 if (!isTypeLegal(VT)) in RISCVTargetLowering()
1023 SetCommonVFPActions(VT); in RISCVTargetLowering()
1026 for (MVT VT : F16VecVTs) { in RISCVTargetLowering() local
1027 if (!isTypeLegal(VT)) in RISCVTargetLowering()
1029 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1030 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT, in RISCVTargetLowering()
1032 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1033 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT, in RISCVTargetLowering()
1035 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1038 VT, Custom); in RISCVTargetLowering()
1041 VT, Custom); in RISCVTargetLowering()
1042 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
1044 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1047 if (VT == MVT::nxv32f16) { in RISCVTargetLowering()
1048 setOperationAction(ZvfhminPromoteOps, VT, Custom); in RISCVTargetLowering()
1049 setOperationAction(ZvfhminPromoteVPOps, VT, Custom); in RISCVTargetLowering()
1053 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in RISCVTargetLowering()
1054 setOperationPromotedToType(ZvfhminPromoteOps, VT, F32VecVT); in RISCVTargetLowering()
1055 setOperationPromotedToType(ZvfhminPromoteVPOps, VT, F32VecVT); in RISCVTargetLowering()
1060 for (MVT VT : F32VecVTs) { in RISCVTargetLowering() local
1061 if (!isTypeLegal(VT)) in RISCVTargetLowering()
1063 SetCommonVFPActions(VT); in RISCVTargetLowering()
1064 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs); in RISCVTargetLowering()
1069 for (MVT VT : F64VecVTs) { in RISCVTargetLowering() local
1070 if (!isTypeLegal(VT)) in RISCVTargetLowering()
1072 SetCommonVFPActions(VT); in RISCVTargetLowering()
1073 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs); in RISCVTargetLowering()
1074 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs); in RISCVTargetLowering()
1079 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) { in RISCVTargetLowering() local
1080 if (!useRVVForFixedLengthVectorVT(VT)) in RISCVTargetLowering()
1085 setOperationAction(Op, VT, Expand); in RISCVTargetLowering()
1087 setTruncStoreAction(VT, OtherVT, Expand); in RISCVTargetLowering()
1088 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
1094 setOperationAction(ISD::UNDEF, VT, Custom); in RISCVTargetLowering()
1097 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
1100 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT, in RISCVTargetLowering()
1104 VT, Custom); in RISCVTargetLowering()
1106 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in RISCVTargetLowering()
1108 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1110 setOperationAction(ISD::SETCC, VT, Custom); in RISCVTargetLowering()
1112 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
1114 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
1116 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
1119 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
1123 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
1137 VT, Custom); in RISCVTargetLowering()
1138 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
1141 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in RISCVTargetLowering()
1144 if (VT.getVectorElementType() == MVT::i1) { in RISCVTargetLowering()
1147 VT, Custom); in RISCVTargetLowering()
1151 VT, Custom); in RISCVTargetLowering()
1153 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); in RISCVTargetLowering()
1154 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); in RISCVTargetLowering()
1163 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) { in RISCVTargetLowering()
1164 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
1165 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
1169 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom); in RISCVTargetLowering()
1175 VT, Custom); in RISCVTargetLowering()
1180 VT, Custom); in RISCVTargetLowering()
1183 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom); in RISCVTargetLowering()
1186 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) in RISCVTargetLowering()
1187 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom); in RISCVTargetLowering()
1191 VT, Custom); in RISCVTargetLowering()
1193 setOperationAction(ISD::VSELECT, VT, Custom); in RISCVTargetLowering()
1194 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1197 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom); in RISCVTargetLowering()
1204 VT, Custom); in RISCVTargetLowering()
1206 setOperationAction(IntegerVPOps, VT, Custom); in RISCVTargetLowering()
1209 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom); in RISCVTargetLowering()
1214 VT, Custom); in RISCVTargetLowering()
1218 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in RISCVTargetLowering()
1221 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT, in RISCVTargetLowering()
1226 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) { in RISCVTargetLowering() local
1229 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in RISCVTargetLowering()
1230 setTruncStoreAction(VT, InnerVT, Expand); in RISCVTargetLowering()
1233 if (!useRVVForFixedLengthVectorVT(VT)) in RISCVTargetLowering()
1238 setOperationAction(Op, VT, Expand); in RISCVTargetLowering()
1242 setOperationAction(ISD::UNDEF, VT, Custom); in RISCVTargetLowering()
1244 if (VT.getVectorElementType() == MVT::f16 && in RISCVTargetLowering()
1246 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1247 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT, in RISCVTargetLowering()
1249 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1251 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT, in RISCVTargetLowering()
1255 VT, Custom); in RISCVTargetLowering()
1258 VT, Custom); in RISCVTargetLowering()
1259 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1260 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
1261 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in RISCVTargetLowering()
1267 setOperationPromotedToType(ZvfhminPromoteOps, VT, F32VecVT); in RISCVTargetLowering()
1268 setOperationPromotedToType(ZvfhminPromoteVPOps, VT, F32VecVT); in RISCVTargetLowering()
1273 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
1279 VT, Custom); in RISCVTargetLowering()
1283 VT, Custom); in RISCVTargetLowering()
1289 VT, Custom); in RISCVTargetLowering()
1295 VT, Custom); in RISCVTargetLowering()
1297 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1301 VT, Custom); in RISCVTargetLowering()
1303 setCondCodeAction(VFPCCToExpand, VT, Expand); in RISCVTargetLowering()
1305 setOperationAction(ISD::SETCC, VT, Custom); in RISCVTargetLowering()
1306 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom); in RISCVTargetLowering()
1307 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1309 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
1311 setOperationAction(FloatingPointVecReduceOps, VT, Custom); in RISCVTargetLowering()
1313 setOperationAction(FloatingPointVPOps, VT, Custom); in RISCVTargetLowering()
1315 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT, in RISCVTargetLowering()
1323 VT, Custom); in RISCVTargetLowering()
1420 EVT VT) const { in getSetCCResultType()
1421 if (!VT.isVector()) in getSetCCResultType()
1424 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors())) in getSetCCResultType()
1425 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount()); in getSetCCResultType()
1426 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
1835 EVT VT = Y.getValueType(); in hasAndNotCompare() local
1838 if (VT.isVector()) in hasAndNotCompare()
1858 EVT VT) const { in shouldFoldSelectWithIdentityConstant()
1860 if (!VT.isVector() || !Subtarget.hasVInstructions()) in shouldFoldSelectWithIdentityConstant()
1863 if (VT.isFixedLengthVector() && !isTypeLegal(VT)) in shouldFoldSelectWithIdentityConstant()
2077 EVT VT) const { in getLegalZfaFPImm()
2082 if (VT == MVT::f16) { in getLegalZfaFPImm()
2084 } else if (VT == MVT::f32) { in getLegalZfaFPImm()
2086 } else if (VT == MVT::f64) { in getLegalZfaFPImm()
2102 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
2105 if (VT == MVT::f16) in isFPImmLegal()
2107 else if (VT == MVT::f32) in isFPImmLegal()
2109 else if (VT == MVT::f64) in isFPImmLegal()
2111 else if (VT == MVT::bf16) in isFPImmLegal()
2117 if (getLegalZfaFPImm(Imm, VT).first >= 0) in isFPImmLegal()
2121 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) { in isFPImmLegal()
2174 EVT VT) const { in getRegisterTypeForCallingConv()
2177 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getRegisterTypeForCallingConv()
2181 MVT PartVT = TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); in getRegisterTypeForCallingConv()
2191 EVT VT) const { in getNumRegistersForCallingConv()
2194 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getNumRegistersForCallingConv()
2198 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); in getNumRegistersForCallingConv()
2202 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, in getVectorTypeBreakdownForCallingConv() argument
2205 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); in getVectorTypeBreakdownForCallingConv()
2282 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) { in getLMUL() argument
2283 assert(VT.isScalableVector() && "Expecting a scalable vector type"); in getLMUL()
2284 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue(); in getLMUL()
2285 if (VT.getVectorElementType() == MVT::i1) in getLMUL()
2326 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) { in getSubregIndexByMVT() argument
2327 RISCVII::VLMUL LMUL = getLMUL(VT); in getSubregIndexByMVT()
2349 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) { in getRegClassIDForVecVT() argument
2350 if (VT.getVectorElementType() == MVT::i1) in getRegClassIDForVecVT()
2352 return getRegClassIDForLMUL(getLMUL(VT)); in getRegClassIDForVecVT()
2393 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const { in mergeStoresAfterLegalization()
2395 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1); in mergeStoresAfterLegalization()
2439 static bool useRVVForFixedLengthVectorVT(MVT VT, in useRVVForFixedLengthVectorVT() argument
2441 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!"); in useRVVForFixedLengthVectorVT()
2449 if (VT.getFixedSizeInBits() > 1024 * 8) in useRVVForFixedLengthVectorVT()
2454 MVT EltVT = VT.getVectorElementType(); in useRVVForFixedLengthVectorVT()
2463 if (VT.getVectorNumElements() > MinVLen) in useRVVForFixedLengthVectorVT()
2493 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen); in useRVVForFixedLengthVectorVT()
2500 if (!VT.isPow2VectorType()) in useRVVForFixedLengthVectorVT()
2506 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const { in useRVVForFixedLengthVectorVT()
2507 return ::useRVVForFixedLengthVectorVT(VT, Subtarget); in useRVVForFixedLengthVectorVT()
2511 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT, in getContainerForFixedLengthVector() argument
2514 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) || in getContainerForFixedLengthVector()
2515 useRVVForFixedLengthVectorVT(VT, Subtarget)) && in getContainerForFixedLengthVector()
2521 MVT EltVT = VT.getVectorElementType(); in getContainerForFixedLengthVector()
2537 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen; in getContainerForFixedLengthVector()
2545 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT, in getContainerForFixedLengthVector() argument
2547 return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT, in getContainerForFixedLengthVector()
2551 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const { in getContainerForFixedLengthVector()
2552 return ::getContainerForFixedLengthVector(*this, VT, getSubtarget()); in getContainerForFixedLengthVector()
2556 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG, in convertToScalableVector() argument
2558 assert(VT.isScalableVector() && in convertToScalableVector()
2564 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
2568 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG, in convertFromScalableVector() argument
2570 assert(VT.isFixedLengthVector() && in convertFromScalableVector()
2576 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
2676 EVT VT, unsigned DefinedValues) const { in shouldExpandBuildVectorWithShuffles() argument
2680 InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const { in getLMULCost()
2683 if (!VT.isVector()) in getLMULCost()
2687 if (VT.isScalableVector()) { in getLMULCost()
2691 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT)); in getLMULCost()
2697 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor); in getLMULCost()
2706 InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const { in getVRGatherVVCost()
2707 return getLMULCost(VT) * getLMULCost(VT); in getVRGatherVVCost()
2713 InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const { in getVRGatherVICost()
2714 return getLMULCost(VT); in getVRGatherVICost()
2721 InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const { in getVSlideVXCost()
2722 return getLMULCost(VT); in getVSlideVXCost()
2729 InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const { in getVSlideVICost()
2730 return getLMULCost(VT); in getVSlideVICost()
2870 MVT VT = Op.getSimpleValueType(); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND() local
2871 assert(VT.isVector() && "Unexpected type"); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2877 MVT ContainerVT = VT; in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2878 if (VT.isFixedLengthVector()) { in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2879 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2886 if (VT.isFixedLengthVector()) in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2891 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2967 if (!VT.isFixedLengthVector()) in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2970 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2980 MVT VT = Op.getSimpleValueType(); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND() local
2984 MVT ContainerVT = VT; in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2985 if (VT.isFixedLengthVector()) { in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2986 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2990 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3072 if (VT.isFixedLengthVector()) in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3073 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3080 MVT VT = Op.getSimpleValueType(); in lowerFTRUNC_FCEIL_FFLOOR_FROUND() local
3081 if (VT.isVector()) in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3093 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3098 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3101 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode, in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3108 MVT VT = Op.getSimpleValueType(); in lowerVectorXRINT() local
3109 assert(VT.isVector() && "Unexpected type"); in lowerVectorXRINT()
3113 MVT ContainerVT = VT; in lowerVectorXRINT()
3115 if (VT.isFixedLengthVector()) { in lowerVectorXRINT()
3116 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorXRINT()
3120 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorXRINT()
3124 if (!VT.isFixedLengthVector()) in lowerVectorXRINT()
3127 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorXRINT()
3132 const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, in getVSlidedown() argument
3139 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops); in getVSlidedown()
3144 EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, in getVSlideup() argument
3151 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops); in getVSlideup()
3154 static MVT getLMUL1VT(MVT VT) { in getLMUL1VT() argument
3155 assert(VT.getVectorElementType().getSizeInBits() <= 64 && in getLMUL1VT()
3158 VT.getVectorElementType(), in getLMUL1VT()
3159 RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits()); in getLMUL1VT()
3303 static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, in matchSplatAsGather() argument
3312 if (Vec.getValueType() != VT || VT.getVectorElementType() == MVT::i1) in matchSplatAsGather()
3319 MVT ContainerVT = VT; in matchSplatAsGather()
3320 if (VT.isFixedLengthVector()) { in matchSplatAsGather()
3321 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in matchSplatAsGather()
3325 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in matchSplatAsGather()
3330 if (!VT.isFixedLengthVector()) in matchSplatAsGather()
3333 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in matchSplatAsGather()
3347 MVT VT = Op.getSimpleValueType(); in lowerBuildVectorViaDominantValues() local
3348 assert(VT.isFixedLengthVector() && "Unexpected vector!"); in lowerBuildVectorViaDominantValues()
3350 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorViaDominantValues()
3353 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3401 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue); in lowerBuildVectorViaDominantValues()
3415 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL; in lowerBuildVectorViaDominantValues()
3416 if (!VT.isFloatingPoint()) in lowerBuildVectorViaDominantValues()
3420 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3424 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1); in lowerBuildVectorViaDominantValues()
3430 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V, in lowerBuildVectorViaDominantValues()
3440 Vec = DAG.getNode(ISD::VSELECT, DL, VT, in lowerBuildVectorViaDominantValues()
3442 DAG.getSplatBuildVector(VT, DL, V), Vec); in lowerBuildVectorViaDominantValues()
3454 MVT VT = Op.getSimpleValueType(); in lowerBuildVectorOfConstants() local
3455 assert(VT.isFixedLengthVector() && "Unexpected vector!"); in lowerBuildVectorOfConstants()
3457 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorOfConstants()
3460 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorOfConstants()
3465 if (VT.getVectorElementType() == MVT::i1) { in lowerBuildVectorOfConstants()
3468 return convertFromScalableVector(VT, VMClr, DAG, Subtarget); in lowerBuildVectorOfConstants()
3473 return convertFromScalableVector(VT, VMSet, DAG, Subtarget); in lowerBuildVectorOfConstants()
3530 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec, in lowerBuildVectorOfConstants()
3535 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits()); in lowerBuildVectorOfConstants()
3536 Vec = DAG.getBitcast(VT, Vec); in lowerBuildVectorOfConstants()
3543 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL in lowerBuildVectorOfConstants()
3545 if (!VT.isFloatingPoint()) in lowerBuildVectorOfConstants()
3549 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBuildVectorOfConstants()
3582 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT; in lowerBuildVectorOfConstants()
3606 if (VT.isFloatingPoint()) { in lowerBuildVectorOfConstants()
3608 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID); in lowerBuildVectorOfConstants()
3616 const unsigned EltBitSize = VT.getScalarSizeInBits(); in lowerBuildVectorOfConstants()
3617 if (VT.getSizeInBits() <= 32 && in lowerBuildVectorOfConstants()
3619 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits()); in lowerBuildVectorOfConstants()
3626 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1; in lowerBuildVectorOfConstants()
3652 return DAG.getBitcast(VT, Vec); in lowerBuildVectorOfConstants()
3665 if (VT.isInteger() && EltBitSize < Subtarget.getELen() && in lowerBuildVectorOfConstants()
3721 return DAG.getBitcast(VT, Splat); in lowerBuildVectorOfConstants()
3731 if (EltBitSize > 8 && VT.isInteger() && in lowerBuildVectorOfConstants()
3732 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen())) { in lowerBuildVectorOfConstants()
3735 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8), in lowerBuildVectorOfConstants()
3740 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerBuildVectorOfConstants()
3755 MVT VT = Op.getSimpleValueType(); in lowerBUILD_VECTOR() local
3756 assert(VT.isFixedLengthVector() && "Unexpected vector!"); in lowerBUILD_VECTOR()
3762 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
3765 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBUILD_VECTOR()
3769 if (VT.getVectorElementType() == MVT::i1) { in lowerBUILD_VECTOR()
3773 MVT WideVecVT = VT.changeVectorElementType(MVT::i8); in lowerBUILD_VECTOR()
3790 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE); in lowerBUILD_VECTOR()
3794 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget)) in lowerBUILD_VECTOR()
3796 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL in lowerBUILD_VECTOR()
3798 if (!VT.isFloatingPoint()) in lowerBUILD_VECTOR()
3802 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBUILD_VECTOR()
3812 if (MinVLen == MaxVLen && VT.getSizeInBits().getKnownMinValue() > MinVLen) { in lowerBUILD_VECTOR()
3813 MVT ElemVT = VT.getVectorElementType(); in lowerBUILD_VECTOR()
3815 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
3827 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) { in lowerBUILD_VECTOR()
3836 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
3846 InstructionCost LinearBudget = VT.getVectorNumElements() * 2; in lowerBUILD_VECTOR()
3884 assert((!VT.isFloatingPoint() || in lowerBUILD_VECTOR()
3885 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) && in lowerBUILD_VECTOR()
3902 Vec = DAG.getSplatVector(VT, DL, V); in lowerBUILD_VECTOR()
3915 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL; in lowerBUILD_VECTOR()
3916 if (!VT.isFloatingPoint()) in lowerBUILD_VECTOR()
3926 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
3929 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, in splatPartsI64WithVL() argument
3933 Passthru = DAG.getUNDEF(VT); in splatPartsI64WithVL()
3940 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3958 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2); in splatPartsI64WithVL()
3962 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec); in splatPartsI64WithVL()
3971 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3976 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3979 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo, in splatPartsI64WithVL()
3986 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, in splatSplitI64WithVL() argument
3992 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG); in splatSplitI64WithVL()
3999 MVT VT, const SDLoc &DL, SelectionDAG &DAG, in lowerScalarSplat() argument
4003 Passthru = DAG.getUNDEF(VT); in lowerScalarSplat()
4004 if (VT.isFloatingPoint()) in lowerScalarSplat()
4005 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL); in lowerScalarSplat()
4018 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL); in lowerScalarSplat()
4025 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, in lowerScalarSplat()
4029 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG); in lowerScalarSplat()
4036 static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, in lowerScalarInsert() argument
4039 assert(VT.isScalableVector() && "Expect VT is scalable vector type."); in lowerScalarInsert()
4042 SDValue Passthru = DAG.getUNDEF(VT); in lowerScalarInsert()
4049 VT.getVectorElementType()) { in lowerScalarInsert()
4058 if (ExtractedContainerVT.bitsLE(VT)) in lowerScalarInsert()
4059 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, in lowerScalarInsert()
4061 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal, in lowerScalarInsert()
4067 if (VT.isFloatingPoint()) in lowerScalarInsert()
4068 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, in lowerScalarInsert()
4069 DAG.getUNDEF(VT), Scalar, VL); in lowerScalarInsert()
4074 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar, in lowerScalarInsert()
4076 VT, DL, DAG, Subtarget); in lowerScalarInsert()
4085 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, in lowerScalarInsert()
4086 DAG.getUNDEF(VT), Scalar, VL); in lowerScalarInsert()
4096 static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, in isDeinterleaveShuffle() argument
4100 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isDeinterleaveShuffle()
4141 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc, in isInterleaveShuffle() argument
4144 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isInterleaveShuffle()
4148 int NumElts = VT.getVectorNumElements(); in isInterleaveShuffle()
4251 static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, in getDeinterleaveViaVNSRL() argument
4256 MVT ContainerVT = VT; in getDeinterleaveViaVNSRL()
4269 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4293 if (VT.isFixedLengthVector()) in getDeinterleaveViaVNSRL()
4294 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4310 static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlidedown() argument
4370 ISD::EXTRACT_SUBVECTOR, DL, VT, in lowerVECTOR_SHUFFLEAsVSlidedown()
4387 static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlideup() argument
4392 unsigned NumElts = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLEAsVSlideup()
4403 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4404 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first; in lowerVECTOR_SHUFFLEAsVSlideup()
4426 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4431 static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlide1() argument
4458 const unsigned NumElts = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLEAsVSlide1()
4468 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4469 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4471 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) : in lowerVECTOR_SHUFFLEAsVSlide1()
4472 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL); in lowerVECTOR_SHUFFLEAsVSlide1()
4473 if (!VT.isFloatingPoint()) in lowerVECTOR_SHUFFLEAsVSlide1()
4479 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4573 MVT VT = SVN->getSimpleValueType(0); in lowerBitreverseShuffle() local
4575 unsigned NumElts = VT.getVectorNumElements(); in lowerBitreverseShuffle()
4577 assert(VT.getVectorElementType() == MVT::i1); in lowerBitreverseShuffle()
4616 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerBitreverseShuffle()
4629 EVT VT = SVN->getValueType(0); in lowerVECTOR_SHUFFLEAsRotate() local
4630 unsigned NumElts = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLEAsRotate()
4631 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in lowerVECTOR_SHUFFLEAsRotate()
4654 return DAG.getBitcast(VT, Rotate); in lowerVECTOR_SHUFFLEAsRotate()
4664 MVT VT = SVN->getSimpleValueType(0); in lowerShuffleViaVRegSplitting() local
4668 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleViaVRegSplitting()
4674 if (MinVLen != MaxVLen || VT.getSizeInBits().getFixedValue() <= MinVLen) in lowerShuffleViaVRegSplitting()
4677 MVT ElemVT = VT.getVectorElementType(); in lowerShuffleViaVRegSplitting()
4707 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerShuffleViaVRegSplitting()
4733 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4742 MVT VT = Op.getSimpleValueType(); in lowerVECTOR_SHUFFLE() local
4743 unsigned NumElts = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLE()
4746 if (VT.getVectorElementType() == MVT::i1) { in lowerVECTOR_SHUFFLE()
4755 MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()); in lowerVECTOR_SHUFFLE()
4760 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT), in lowerVECTOR_SHUFFLE()
4764 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLE()
4766 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4771 MVT SVT = VT.getVectorElementType(); in lowerVECTOR_SHUFFLE()
4809 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4828 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL; in lowerVECTOR_SHUFFLE()
4831 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4839 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4851 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4855 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4895 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4900 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) { in lowerVECTOR_SHUFFLE()
4901 return getDeinterleaveViaVNSRL(DL, VT, V1.getOperand(0), Mask[0] == 0, in lowerVECTOR_SHUFFLE()
4906 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4912 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget)) { in lowerVECTOR_SHUFFLE()
4914 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in lowerVECTOR_SHUFFLE()
4961 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2); in lowerVECTOR_SHUFFLE()
4970 if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) { in lowerVECTOR_SHUFFLE()
5017 MVT IndexVT = VT.changeTypeToInteger(); in lowerVECTOR_SHUFFLE()
5090 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5093 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { in isShuffleMaskLegal()
5095 if (ShuffleVectorSDNode::isSplatMask(M.data(), VT)) in isShuffleMaskLegal()
5099 if (!isTypeLegal(VT)) in isShuffleMaskLegal()
5102 MVT SVT = VT.getSimpleVT(); in isShuffleMaskLegal()
5118 MVT VT = Op.getSimpleValueType(); in lowerCTLZ_CTTZ_ZERO_UNDEF() local
5119 unsigned EltSize = VT.getScalarSizeInBits(); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5122 MVT ContainerVT = VT; in lowerCTLZ_CTTZ_ZERO_UNDEF()
5127 if (VT.isFixedLengthVector()) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5137 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount()))) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5139 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount()); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5150 SDValue Neg = DAG.getNegative(Src, DL, VT); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5151 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5153 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT), in lowerCTLZ_CTTZ_ZERO_UNDEF()
5155 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5160 if (FloatVT.bitsGT(VT)) { in lowerCTLZ_CTTZ_ZERO_UNDEF()
5167 if (VT.isFixedLengthVector()) { in lowerCTLZ_CTTZ_ZERO_UNDEF()
5168 ContainerVT = getContainerForFixedLengthVector(VT); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5172 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5179 if (VT.isFixedLengthVector()) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5192 Exp = DAG.getVPZExtOrTrunc(DL, VT, Exp, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5196 if (IntVT.bitsLT(VT)) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5197 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5198 else if (IntVT.bitsGT(VT)) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5199 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5206 return DAG.getNode(ISD::SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5207 DAG.getConstant(ExponentBias, DL, VT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5209 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5210 DAG.getConstant(ExponentBias, DL, VT), Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5217 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5220 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5225 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5227 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5228 DAG.getConstant(EltSize, DL, VT), Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5247 MVT VT = Op.getSimpleValueType(); in expandUnalignedRVVLoad() local
5248 unsigned EltSizeBits = VT.getScalarSizeInBits(); in expandUnalignedRVVLoad()
5252 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8)); in expandUnalignedRVVLoad()
5258 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL); in expandUnalignedRVVLoad()
5278 MVT VT = StoredVal.getSimpleValueType(); in expandUnalignedRVVStore() local
5279 unsigned EltSizeBits = VT.getScalarSizeInBits(); in expandUnalignedRVVStore()
5283 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8)); in expandUnalignedRVVStore()
5364 MVT VT = Op.getSimpleValueType(); in LowerIS_FPCLASS() local
5393 if (VT.isVector()) { in LowerIS_FPCLASS()
5397 if (VT.isScalableVector()) { in LowerIS_FPCLASS()
5407 return DAG.getSetCC(DL, VT, FPCLASS, in LowerIS_FPCLASS()
5412 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT), in LowerIS_FPCLASS()
5417 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerIS_FPCLASS()
5439 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget); in LowerIS_FPCLASS()
5451 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget); in LowerIS_FPCLASS()
5458 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerIS_FPCLASS()
5466 MVT VT = Op.getSimpleValueType(); in lowerFMAXIMUM_FMINIMUM() local
5471 if (!VT.isVector()) { in lowerFMAXIMUM_FMINIMUM()
5482 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X); in lowerFMAXIMUM_FMINIMUM()
5488 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y); in lowerFMAXIMUM_FMINIMUM()
5493 return DAG.getNode(Opc, DL, VT, NewX, NewY); in lowerFMAXIMUM_FMINIMUM()
5500 MVT ContainerVT = VT; in lowerFMAXIMUM_FMINIMUM()
5501 if (VT.isFixedLengthVector()) { in lowerFMAXIMUM_FMINIMUM()
5502 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5510 if (VT.isFixedLengthVector()) in lowerFMAXIMUM_FMINIMUM()
5515 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5542 if (VT.isFixedLengthVector()) in lowerFMAXIMUM_FMINIMUM()
5543 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5885 EVT VT = Op.getValueType(); in LowerOperation() local
5889 if (VT == MVT::f16 && Op0VT == MVT::i16 && in LowerOperation()
5895 if (VT == MVT::bf16 && Op0VT == MVT::i16 && in LowerOperation()
5901 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() && in LowerOperation()
5908 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32 && in LowerOperation()
5919 if (!VT.isVector() && !Op0VT.isVector()) { in LowerOperation()
5920 if (isTypeLegal(VT) && isTypeLegal(Op0VT)) in LowerOperation()
5925 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() && in LowerOperation()
5928 if (VT.isFixedLengthVector()) { in LowerOperation()
5940 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT, in LowerOperation()
5949 if (!VT.isVector() && Op0VT.isFixedLengthVector()) { in LowerOperation()
5950 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1); in LowerOperation()
5954 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in LowerOperation()
5968 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
5969 if (VT.isFixedLengthVector()) { in LowerOperation()
5977 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0)); in LowerOperation()
5978 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap); in LowerOperation()
6003 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6006 if (VT.getVectorElementType() == MVT::i1) { in LowerOperation()
6007 MVT WideVT = VT.changeVectorElementType(MVT::i8); in LowerOperation()
6009 return DAG.getNode(ISD::TRUNCATE, DL, VT, V); in LowerOperation()
6011 MVT ContainerVT = VT; in LowerOperation()
6012 if (VT.isFixedLengthVector()) in LowerOperation()
6013 ContainerVT = getContainerForFixedLengthVector(VT); in LowerOperation()
6014 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerOperation()
6018 if (VT.isFixedLengthVector()) in LowerOperation()
6019 V = convertFromScalableVector(VT, V, DAG, Subtarget); in LowerOperation()
6024 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6040 DAG.getConstant(3 - Log2, DL, VT)); in LowerOperation()
6055 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerOperation()
6080 EVT VT = Op.getValueType(); in LowerOperation() local
6083 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6085 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) { in LowerOperation()
6097 EVT VT = Op.getValueType(); in LowerOperation() local
6100 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6102 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() && in LowerOperation()
6160 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6161 if (!VT.isVector()) in LowerOperation()
6166 MVT EltVT = VT.getVectorElementType(); in LowerOperation()
6180 VT.getVectorElementCount()); in LowerOperation()
6189 return DAG.getNode(Op.getOpcode(), DL, VT, Ext); in LowerOperation()
6195 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in LowerOperation()
6202 return DAG.getNode(Op.getOpcode(), DL, VT, FExt); in LowerOperation()
6210 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount()); in LowerOperation()
6216 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first; in LowerOperation()
6219 return DAG.getFPExtendOrRound(Int2FP, DL, VT); in LowerOperation()
6225 VT.getVectorElementCount()); in LowerOperation()
6230 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6234 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6239 if (!VT.isFixedLengthVector()) in LowerOperation()
6273 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerOperation()
6278 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in LowerOperation()
6284 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6288 return convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6309 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6318 if (VT != MVT::f32) in LowerOperation()
6319 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res); in LowerOperation()
6445 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6448 SDValue Vec = DAG.getUNDEF(VT); in LowerOperation()
6454 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec, in LowerOperation()
6485 EVT VT = Op.getValueType(); in LowerOperation() local
6493 return DAG.getSelect(DL, VT, Cond, True, False); in LowerOperation()
6498 MVT VT = Op.getSimpleValueType(); in LowerOperation() local
6517 return DAG.getConstant(0, DL, VT); in LowerOperation()
6521 DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal); in LowerOperation()
6522 return DAG.getLogicalNOT(DL, SetCC, VT); in LowerOperation()
6529 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal); in LowerOperation()
7063 MVT VT = N->getSimpleValueType(0); in combineSelectToBinOp() local
7069 SDValue Neg = DAG.getNegative(CondV, DL, VT); in combineSelectToBinOp()
7070 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7074 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7075 DAG.getAllOnesConstant(DL, VT)); in combineSelectToBinOp()
7076 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7081 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7082 DAG.getAllOnesConstant(DL, VT)); in combineSelectToBinOp()
7083 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7087 SDValue Neg = DAG.getNegative(CondV, DL, VT); in combineSelectToBinOp()
7088 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7103 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV, in combineSelectToBinOp()
7109 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT, TrueV, in combineSelectToBinOp()
7156 EVT VT = BO->getValueType(0); in foldBinOpIntoSelectIfProfitable() local
7163 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps); in foldBinOpIntoSelectIfProfitable()
7175 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps); in foldBinOpIntoSelectIfProfitable()
7179 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF); in foldBinOpIntoSelectIfProfitable()
7187 MVT VT = Op.getSimpleValueType(); in lowerSELECT() local
7191 if (VT.isVector()) { in lowerSELECT()
7192 MVT SplatCondVT = VT.changeVectorElementType(MVT::i1); in lowerSELECT()
7194 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV); in lowerSELECT()
7202 VT.isScalarInteger()) { in lowerSELECT()
7205 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV); in lowerSELECT()
7208 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV); in lowerSELECT()
7214 ISD::OR, DL, VT, TrueV, in lowerSELECT()
7215 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV)); in lowerSELECT()
7220 ISD::OR, DL, VT, FalseV, in lowerSELECT()
7221 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV)); in lowerSELECT()
7231 ISD::OR, DL, VT, in lowerSELECT()
7232 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV), in lowerSELECT()
7233 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV)); in lowerSELECT()
7257 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV); in lowerSELECT()
7261 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR); in lowerSELECT()
7276 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops); in lowerSELECT()
7300 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV); in lowerSELECT()
7302 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV); in lowerSELECT()
7309 LHS = DAG.getConstant(0, DL, VT); in lowerSELECT()
7320 RHS = DAG.getConstant(0, DL, VT); in lowerSELECT()
7333 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops); in lowerSELECT()
7383 EVT VT = Op.getValueType(); in lowerFRAMEADDR() local
7385 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT); in lowerFRAMEADDR()
7389 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, in lowerFRAMEADDR()
7392 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); in lowerFRAMEADDR()
7409 EVT VT = Op.getValueType(); in lowerRETURNADDR() local
7415 SDValue Offset = DAG.getConstant(Off, DL, VT); in lowerRETURNADDR()
7416 return DAG.getLoad(VT, DL, DAG.getEntryNode(), in lowerRETURNADDR()
7417 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in lowerRETURNADDR()
7433 EVT VT = Lo.getValueType(); in lowerShiftLeftParts() local
7442 SDValue Zero = DAG.getConstant(0, DL, VT); in lowerShiftLeftParts()
7443 SDValue One = DAG.getConstant(1, DL, VT); in lowerShiftLeftParts()
7444 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftLeftParts()
7445 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftLeftParts()
7446 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftLeftParts()
7447 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftLeftParts()
7449 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); in lowerShiftLeftParts()
7450 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); in lowerShiftLeftParts()
7452 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt); in lowerShiftLeftParts()
7453 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); in lowerShiftLeftParts()
7454 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); in lowerShiftLeftParts()
7455 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen); in lowerShiftLeftParts()
7457 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftLeftParts()
7459 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); in lowerShiftLeftParts()
7460 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftLeftParts()
7472 EVT VT = Lo.getValueType(); in lowerShiftRightParts() local
7492 SDValue Zero = DAG.getConstant(0, DL, VT); in lowerShiftRightParts()
7493 SDValue One = DAG.getConstant(1, DL, VT); in lowerShiftRightParts()
7494 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftRightParts()
7495 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftRightParts()
7496 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftRightParts()
7497 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftRightParts()
7499 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); in lowerShiftRightParts()
7500 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); in lowerShiftRightParts()
7502 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt); in lowerShiftRightParts()
7503 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); in lowerShiftRightParts()
7504 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt); in lowerShiftRightParts()
7505 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen); in lowerShiftRightParts()
7507 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero; in lowerShiftRightParts()
7509 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftRightParts()
7511 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); in lowerShiftRightParts()
7512 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftRightParts()
7523 MVT VT = Op.getSimpleValueType(); in lowerVectorMaskSplat() local
7527 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7528 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL); in lowerVectorMaskSplat()
7531 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7532 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL); in lowerVectorMaskSplat()
7534 MVT InterVT = VT.changeVectorElementType(MVT::i8); in lowerVectorMaskSplat()
7539 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE); in lowerVectorMaskSplat()
7620 MVT VT = Op.getOperand(0).getSimpleValueType(); in lowerFixedLengthVectorExtendToRVV() local
7627 MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(), in lowerFixedLengthVectorExtendToRVV()
7634 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7701 MVT VT = Op.getSimpleValueType(); in lowerVectorTruncLike() local
7703 assert(VT.isVector() && "Unexpected type for vector truncate lowering"); in lowerVectorTruncLike()
7706 if (VT.getVectorElementType() == MVT::i1) in lowerVectorTruncLike()
7712 MVT DstEltVT = VT.getVectorElementType(); in lowerVectorTruncLike()
7753 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorTruncLike()
7764 MVT VT = Op.getSimpleValueType(); in lowerStrictFPExtendOrRoundLike() local
7766 MVT ContainerVT = VT; in lowerStrictFPExtendOrRoundLike()
7767 if (VT.isFixedLengthVector()) { in lowerStrictFPExtendOrRoundLike()
7770 SrcContainerVT.changeVectorElementType(VT.getVectorElementType()); in lowerStrictFPExtendOrRoundLike()
7777 if ((VT.getVectorElementType() == MVT::f64 && in lowerStrictFPExtendOrRoundLike()
7779 (VT.getVectorElementType() == MVT::f16 && in lowerStrictFPExtendOrRoundLike()
7796 if (VT.isFixedLengthVector()) { in lowerStrictFPExtendOrRoundLike()
7799 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7816 MVT VT = Op.getSimpleValueType(); in lowerVectorFPExtendOrRoundLike() local
7818 assert(VT.isVector() && "Unexpected type for vector truncate lowering"); in lowerVectorFPExtendOrRoundLike()
7823 bool IsDirectExtend = IsExtend && (VT.getVectorElementType() != MVT::f64 || in lowerVectorFPExtendOrRoundLike()
7825 bool IsDirectTrunc = !IsExtend && (VT.getVectorElementType() != MVT::f16 || in lowerVectorFPExtendOrRoundLike()
7831 MVT ContainerVT = VT; in lowerVectorFPExtendOrRoundLike()
7837 if (VT.isFixedLengthVector()) { in lowerVectorFPExtendOrRoundLike()
7840 SrcContainerVT.changeVectorElementType(VT.getVectorElementType()); in lowerVectorFPExtendOrRoundLike()
7856 if (VT.isFixedLengthVector()) in lowerVectorFPExtendOrRoundLike()
7857 Src = convertFromScalableVector(VT, Src, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7869 if (VT.isFixedLengthVector()) in lowerVectorFPExtendOrRoundLike()
7870 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8280 MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType(); in lowerVectorIntrinsicScalars() local
8284 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!"); in lowerVectorIntrinsicScalars()
8303 MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2); in lowerVectorIntrinsicScalars()
8316 RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget); in lowerVectorIntrinsicScalars()
8338 RISCVII::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT); in lowerVectorIntrinsicScalars()
8340 unsigned Sew = RISCVVType::encodeSEW(VT.getScalarSizeInBits()); in lowerVectorIntrinsicScalars()
8376 Vec = DAG.getBitcast(VT, Vec); in lowerVectorIntrinsicScalars()
8390 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff, in lowerVectorIntrinsicScalars()
8391 DAG.getUNDEF(VT), AVL); in lowerVectorIntrinsicScalars()
8394 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff, in lowerVectorIntrinsicScalars()
8402 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG); in lowerVectorIntrinsicScalars()
8474 static inline bool isValidEGW(int EGS, EVT VT, in isValidEGW() argument
8477 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >= in isValidEGW()
8478 EGS * VT.getScalarSizeInBits(); in isValidEGW()
8621 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
8625 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG); in LowerINTRINSIC_WO_CHAIN()
8629 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in LowerINTRINSIC_WO_CHAIN()
8632 MVT MaskVT = getMaskTypeFor(VT); in LowerINTRINSIC_WO_CHAIN()
8633 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG); in LowerINTRINSIC_WO_CHAIN()
8634 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL); in LowerINTRINSIC_WO_CHAIN()
8639 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal, in LowerINTRINSIC_WO_CHAIN()
8640 Vec, DAG.getUNDEF(VT), VL); in LowerINTRINSIC_WO_CHAIN()
8701 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
8706 MVT RetVT = VT; in LowerINTRINSIC_WO_CHAIN()
8707 if (VT.isFixedLengthVector()) in LowerINTRINSIC_WO_CHAIN()
8708 RetVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_WO_CHAIN()
8709 else if (VT.isFloatingPoint()) in LowerINTRINSIC_WO_CHAIN()
8710 RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()), in LowerINTRINSIC_WO_CHAIN()
8711 VT.getVectorElementCount()); in LowerINTRINSIC_WO_CHAIN()
8715 if (VT.isFixedLengthVector()) in LowerINTRINSIC_WO_CHAIN()
8716 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
8717 else if (VT.isFloatingPoint()) in LowerINTRINSIC_WO_CHAIN()
8718 NewNode = DAG.getBitcast(VT, NewNode); in LowerINTRINSIC_WO_CHAIN()
8745 MVT VT = Op->getSimpleValueType(0); in LowerINTRINSIC_W_CHAIN() local
8746 MVT ContainerVT = VT; in LowerINTRINSIC_W_CHAIN()
8747 if (VT.isFixedLengthVector()) in LowerINTRINSIC_W_CHAIN()
8748 ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_W_CHAIN()
8753 if (VT.isFixedLengthVector()) { in LowerINTRINSIC_W_CHAIN()
8760 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_W_CHAIN()
8802 if (VT.isFixedLengthVector()) in LowerINTRINSIC_W_CHAIN()
8803 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8822 MVT VT = Op->getSimpleValueType(0); in LowerINTRINSIC_W_CHAIN() local
8823 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_W_CHAIN()
8825 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_W_CHAIN()
8841 Results.push_back(convertFromScalableVector(VT, Result.getValue(RetIdx), in LowerINTRINSIC_W_CHAIN()
8860 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
8865 MVT RetVT = VT; in LowerINTRINSIC_W_CHAIN()
8866 if (VT.isFixedLengthVector()) in LowerINTRINSIC_W_CHAIN()
8867 RetVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_W_CHAIN()
8868 else if (VT.isFloatingPoint()) in LowerINTRINSIC_W_CHAIN()
8875 if (VT.isFixedLengthVector()) { in LowerINTRINSIC_W_CHAIN()
8877 convertFromScalableVector(VT, NewNode, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8879 } else if (VT.isFloatingPoint()) { in LowerINTRINSIC_W_CHAIN()
8880 SDValue BitCast = DAG.getBitcast(VT, NewNode.getValue(0)); in LowerINTRINSIC_W_CHAIN()
8910 MVT VT = Val.getSimpleValueType(); in LowerINTRINSIC_VOID() local
8911 MVT ContainerVT = VT; in LowerINTRINSIC_VOID()
8912 if (VT.isFixedLengthVector()) { in LowerINTRINSIC_VOID()
8913 ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_VOID()
8918 if (VT.isFixedLengthVector()) in LowerINTRINSIC_VOID()
8922 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_VOID()
8958 MVT VT = Op->getOperand(2).getSimpleValueType(); in LowerINTRINSIC_VOID() local
8959 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_VOID()
8961 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_VOID()
9659 MVT VT = N.getSimpleValueType(); in widenVectorOpsToi8() local
9660 MVT WideVT = VT.changeVectorElementType(MVT::i8); in widenVectorOpsToi8()
9663 assert(Op.getSimpleValueType() == VT && in widenVectorOpsToi8()
9856 MVT VT = Op.getSimpleValueType(); in lowerSTEP_VECTOR() local
9857 assert(VT.isScalableVector() && "Expected scalable vector"); in lowerSTEP_VECTOR()
9859 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9860 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL); in lowerSTEP_VECTOR()
9865 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in lowerSTEP_VECTOR()
9867 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
9870 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), in lowerSTEP_VECTOR()
9871 VL, VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9872 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
10002 MVT VT = Op.getSimpleValueType(); in lowerFixedLengthVectorLoadToRVV() local
10004 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorLoadToRVV()
10010 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && in lowerFixedLengthVectorLoadToRVV()
10015 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10019 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10021 bool IsMaskOp = VT.getVectorElementType() == MVT::i1; in lowerFixedLengthVectorLoadToRVV()
10034 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10050 MVT VT = StoreVal.getSimpleValueType(); in lowerFixedLengthVectorStoreToRVV() local
10054 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) { in lowerFixedLengthVectorStoreToRVV()
10055 VT = MVT::v8i1; in lowerFixedLengthVectorStoreToRVV()
10056 StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerFixedLengthVectorStoreToRVV()
10057 DAG.getConstant(0, DL, VT), StoreVal, in lowerFixedLengthVectorStoreToRVV()
10061 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorStoreToRVV()
10071 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && in lowerFixedLengthVectorStoreToRVV()
10076 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in lowerFixedLengthVectorStoreToRVV()
10079 bool IsMaskOp = VT.getVectorElementType() == MVT::i1; in lowerFixedLengthVectorStoreToRVV()
10091 MVT VT = Op.getSimpleValueType(); in lowerMaskedLoad() local
10102 PassThru = DAG.getUNDEF(VT); in lowerMaskedLoad()
10114 MVT ContainerVT = VT; in lowerMaskedLoad()
10115 if (VT.isFixedLengthVector()) { in lowerMaskedLoad()
10116 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedLoad()
10125 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedLoad()
10147 if (VT.isFixedLengthVector()) in lowerMaskedLoad()
10148 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedLoad()
10176 MVT VT = Val.getSimpleValueType(); in lowerMaskedStore() local
10179 MVT ContainerVT = VT; in lowerMaskedStore()
10180 if (VT.isFixedLengthVector()) { in lowerMaskedStore()
10181 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedStore()
10191 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedStore()
10212 MVT VT = Op.getSimpleValueType(); in lowerFixedLengthVectorSetccToRVV() local
10220 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL, in lowerFixedLengthVectorSetccToRVV()
10228 return convertFromScalableVector(VT, Cmp, DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10240 MVT VT = Op.getSimpleValueType(); in lowerVectorStrictFSetcc() local
10259 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2); in lowerVectorStrictFSetcc()
10268 SDValue Res = DAG.getNOT(DL, OEQ, VT); in lowerVectorStrictFSetcc()
10312 if (VT.isFixedLengthVector()) { in lowerVectorStrictFSetcc()
10313 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVectorStrictFSetcc()
10322 MVT VT = Op.getSimpleValueType(); in lowerABS() local
10325 assert((Op.getOpcode() == ISD::VP_ABS || VT.isFixedLengthVector()) && in lowerABS()
10328 MVT ContainerVT = VT; in lowerABS()
10329 if (VT.isFixedLengthVector()) { in lowerABS()
10330 ContainerVT = getContainerForFixedLengthVector(VT); in lowerABS()
10337 if (VT.isFixedLengthVector()) in lowerABS()
10342 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerABS()
10352 if (VT.isFixedLengthVector()) in lowerABS()
10353 Max = convertFromScalableVector(VT, Max, DAG, Subtarget); in lowerABS()
10360 MVT VT = Op.getSimpleValueType(); in lowerFixedLengthVectorFCOPYSIGNToRVV() local
10366 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10370 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10375 return convertFromScalableVector(VT, CopySign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10380 MVT VT = Op.getSimpleValueType(); in lowerFixedLengthVectorSelectToRVV() local
10381 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorSelectToRVV()
10394 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerFixedLengthVectorSelectToRVV()
10399 return convertFromScalableVector(VT, Select, DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10408 MVT VT = Op.getSimpleValueType(); in lowerToScalableOp() local
10409 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerToScalableOp()
10429 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerToScalableOp()
10442 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
10448 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
10461 MVT VT = Op.getSimpleValueType(); in lowerVPOp() local
10464 MVT ContainerVT = VT; in lowerVPOp()
10465 if (VT.isFixedLengthVector()) in lowerVPOp()
10466 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPOp()
10503 if (!VT.isFixedLengthVector()) in lowerVPOp()
10504 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags()); in lowerVPOp()
10508 return convertFromScalableVector(VT, VPOp, DAG, Subtarget); in lowerVPOp()
10514 MVT VT = Op.getSimpleValueType(); in lowerVPExtMaskOp() local
10520 MVT ContainerVT = VT; in lowerVPExtMaskOp()
10521 if (VT.isFixedLengthVector()) { in lowerVPExtMaskOp()
10522 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPExtMaskOp()
10539 if (!VT.isFixedLengthVector()) in lowerVPExtMaskOp()
10541 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPExtMaskOp()
10547 MVT VT = Op.getSimpleValueType(); in lowerVPSetCCMaskOp() local
10555 MVT ContainerVT = VT; in lowerVPSetCCMaskOp()
10556 if (VT.isFixedLengthVector()) { in lowerVPSetCCMaskOp()
10557 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPSetCCMaskOp()
10618 if (!VT.isFixedLengthVector()) in lowerVPSetCCMaskOp()
10620 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSetCCMaskOp()
10751 MVT VT = Op.getSimpleValueType(); in lowerVPFPIntConvOp() local
10752 if (!VT.isFixedLengthVector()) in lowerVPFPIntConvOp()
10754 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPFPIntConvOp()
10770 MVT VT = Op.getSimpleValueType(); in lowerVPSpliceExperimental() local
10771 MVT ContainerVT = VT; in lowerVPSpliceExperimental()
10772 if (VT.isFixedLengthVector()) { in lowerVPSpliceExperimental()
10773 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPSpliceExperimental()
10780 bool IsMaskVector = VT.getVectorElementType() == MVT::i1; in lowerVPSpliceExperimental()
10833 if (!VT.isFixedLengthVector()) in lowerVPSpliceExperimental()
10835 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSpliceExperimental()
10842 MVT VT = Op.getSimpleValueType(); in lowerVPReverseExperimental() local
10849 MVT ContainerVT = VT; in lowerVPReverseExperimental()
10850 if (VT.isFixedLengthVector()) { in lowerVPReverseExperimental()
10851 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPReverseExperimental()
10926 if (!VT.isFixedLengthVector()) in lowerVPReverseExperimental()
10928 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
10954 if (!VT.isFixedLengthVector()) in lowerVPReverseExperimental()
10956 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
10961 MVT VT = Op.getSimpleValueType(); in lowerLogicVPOp() local
10962 if (VT.getVectorElementType() != MVT::i1) in lowerLogicVPOp()
10970 MVT ContainerVT = VT; in lowerLogicVPOp()
10971 const bool IsFixed = VT.isFixedLengthVector(); in lowerLogicVPOp()
10973 ContainerVT = getContainerForFixedLengthVector(VT); in lowerLogicVPOp()
10982 return convertFromScalableVector(VT, Val, DAG, Subtarget); in lowerLogicVPOp()
10989 MVT VT = Op.getSimpleValueType(); in lowerVPStridedLoad() local
10990 MVT ContainerVT = VT; in lowerVPStridedLoad()
10991 if (VT.isFixedLengthVector()) in lowerVPStridedLoad()
10992 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPStridedLoad()
11008 if (VT.isFixedLengthVector()) { in lowerVPStridedLoad()
11025 if (VT.isFixedLengthVector()) in lowerVPStridedLoad()
11026 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPStridedLoad()
11038 MVT VT = StoreVal.getSimpleValueType(); in lowerVPStridedStore() local
11039 MVT ContainerVT = VT; in lowerVPStridedStore()
11040 if (VT.isFixedLengthVector()) { in lowerVPStridedStore()
11041 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPStridedStore()
11055 if (VT.isFixedLengthVector()) { in lowerVPStridedStore()
11077 MVT VT = Op.getSimpleValueType(); in lowerMaskedGather() local
11091 PassThru = DAG.getUNDEF(VT); in lowerMaskedGather()
11107 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && in lowerMaskedGather()
11119 MVT ContainerVT = VT; in lowerMaskedGather()
11120 if (VT.isFixedLengthVector()) { in lowerMaskedGather()
11121 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedGather()
11135 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedGather()
11162 if (VT.isFixedLengthVector()) in lowerMaskedGather()
11163 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedGather()
11202 MVT VT = Val.getSimpleValueType(); in lowerMaskedScatter() local
11206 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && in lowerMaskedScatter()
11218 MVT ContainerVT = VT; in lowerMaskedScatter()
11219 if (VT.isFixedLengthVector()) { in lowerMaskedScatter()
11220 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedScatter()
11234 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedScatter()
11623 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
11624 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) && in ReplaceNodeResults()
11638 if (VT != MVT::i32) in ReplaceNodeResults()
11777 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
11778 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!"); in ReplaceNodeResults()
11782 if (VT == MVT::i16 && Op0VT == MVT::f16 && in ReplaceNodeResults()
11786 } else if (VT == MVT::i16 && Op0VT == MVT::bf16 && in ReplaceNodeResults()
11790 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11795 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32 && in ReplaceNodeResults()
11802 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() && in ReplaceNodeResults()
11808 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1); in ReplaceNodeResults()
11811 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in ReplaceNodeResults()
11818 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
11820 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) && in ReplaceNodeResults()
11827 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes)); in ReplaceNodeResults()
11983 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
11985 if (VT.bitsLT(XLenVT)) { in ReplaceNodeResults()
11989 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract)); in ReplaceNodeResults()
11993 assert(VT == MVT::i64 && !Subtarget.is64Bit() && in ReplaceNodeResults()
12103 const EVT VT = N->getValueType(0); in combineBinOpOfExtractToReduceTree() local
12110 if (!VT.isInteger() && in combineBinOpOfExtractToReduceTree()
12133 assert(SrcVecVT.getVectorElementType() == VT); in combineBinOpOfExtractToReduceTree()
12149 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2); in combineBinOpOfExtractToReduceTree()
12152 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags()); in combineBinOpOfExtractToReduceTree()
12171 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1); in combineBinOpOfExtractToReduceTree()
12176 return DAG.getNode(ReduceOpc, DL, VT, Vec, Flags); in combineBinOpOfExtractToReduceTree()
12297 EVT VT = N->getValueType(0); in transformAddShlImm() local
12298 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddShlImm()
12329 DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT)); in transformAddShlImm()
12330 SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS); in transformAddShlImm()
12331 return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT)); in transformAddShlImm()
12349 EVT VT = N->getValueType(0); in combineSelectAndUse() local
12352 if (VT.isVector()) in combineSelectAndUse()
12367 if (VT.getSizeInBits() > Subtarget.getXLen()) in combineSelectAndUse()
12396 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal); in combineSelectAndUse()
12402 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT, in combineSelectAndUse()
12406 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
12444 EVT VT = N->getValueType(0); in transformAddImmMulImm() local
12445 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddImmMulImm()
12483 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0), in transformAddImmMulImm()
12484 DAG.getConstant(CA, DL, VT)); in transformAddImmMulImm()
12486 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT)); in transformAddImmMulImm()
12487 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); in transformAddImmMulImm()
12494 EVT VT = N->getValueType(0); in combineAddOfBooleanXor() local
12506 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1); in combineAddOfBooleanXor()
12511 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineAddOfBooleanXor()
12537 EVT VT = N->getValueType(0); in combineSubOfBoolean() local
12561 DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal); in combineSubOfBoolean()
12570 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT); in combineSubOfBoolean()
12571 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS); in combineSubOfBoolean()
12586 EVT VT = N->getValueType(0); in performSUBCombine() local
12589 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), in performSUBCombine()
12590 DAG.getConstant(ShAmt, DL, VT)); in performSUBCombine()
12629 EVT VT = N->getValueType(0); in combineDeMorganOfBoolean() local
12635 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1); in combineDeMorganOfBoolean()
12642 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10); in combineDeMorganOfBoolean()
12643 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT)); in combineDeMorganOfBoolean()
12649 EVT VT = N->getValueType(0); in performTRUNCATECombine() local
12656 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 && in performTRUNCATECombine()
12663 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl); in performTRUNCATECombine()
12735 EVT VT = N->getValueType(0); in combineOrOfCZERO() local
12738 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0), in combineOrOfCZERO()
12740 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0), in combineOrOfCZERO()
12742 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1); in combineOrOfCZERO()
12743 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1)); in combineOrOfCZERO()
12809 EVT VT = N0.getValueType(); in performXORCombine() local
12813 return DAG.getSetCC(DL, VT, N0.getOperand(1), in performXORCombine()
12814 DAG.getConstant(Imm + 1, DL, VT), CC); in performXORCombine()
12829 EVT VT = N->getValueType(0); in performMULCombine() local
12830 if (!VT.isVector()) in performMULCombine()
12857 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMULCombine()
12858 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal); in performMULCombine()
12862 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMULCombine()
12863 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal); in performMULCombine()
12878 EVT VT = N.getValueType(); in narrowIndex() local
12893 if (ResultVT.bitsLT(VT.getVectorElementType())) { in narrowIndex()
12895 VT.changeVectorElementType(ResultVT), N); in narrowIndex()
12944 EVT VT = N->getValueType(0); in performSETCCCombine() local
12978 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in performSETCCCombine()
12982 return DAG.getSetCC(dl, VT, SExtOp, DAG.getConstant(C1.trunc(32).sext(64), in performSETCCCombine()
12990 EVT VT = N->getValueType(0); in performSIGN_EXTEND_INREGCombine() local
12995 return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT, in performSIGN_EXTEND_INREGCombine()
13113 MVT VT = Root->getSimpleValueType(0); in getNarrowType() local
13116 unsigned NarrowSize = VT.getScalarSizeInBits() / 2; in getNarrowType()
13119 VT.getVectorElementCount()); in getNarrowType()
13193 MVT VT = OrigOperand.getSimpleValueType(); in fillUpExtensionSupport() local
13194 if (!VT.isVector()) in fillUpExtensionSupport()
13200 unsigned ScalarBits = VT.getScalarSizeInBits(); in fillUpExtensionSupport()
13215 std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in fillUpExtensionSupport()
13246 MVT VT = Root->getSimpleValueType(0); in fillUpExtensionSupport() local
13247 unsigned EltBits = VT.getScalarSizeInBits(); in fillUpExtensionSupport()
13254 unsigned NarrowSize = VT.getScalarSizeInBits() / 2; in fillUpExtensionSupport()
13350 MVT VT = Root->getSimpleValueType(0); in getMaskAndVL() local
13351 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in getMaskAndVL()
13869 EVT VT = N->getValueType(0); in performFP_TO_INTCombine() local
13871 if (VT.isVector() && TLI.isTypeLegal(VT)) { in performFP_TO_INTCombine()
13874 MVT ContainerVT = VT.getSimpleVT(); in performFP_TO_INTCombine()
13881 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 || in performFP_TO_INTCombine()
13882 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits()) in performFP_TO_INTCombine()
13915 if (VT.isFixedLengthVector()) in performFP_TO_INTCombine()
13916 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget); in performFP_TO_INTCombine()
13923 if (VT != MVT::i32 && VT != XLenVT) in performFP_TO_INTCombine()
13927 if (VT == XLenVT) in performFP_TO_INTCombine()
13934 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt); in performFP_TO_INTCombine()
14015 EVT VT = N->getValueType(0); in performBITREVERSECombine() local
14016 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() || in performBITREVERSECombine()
14017 !llvm::has_single_bit<uint32_t>(VT.getSizeInBits())) in performBITREVERSECombine()
14021 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0)); in performBITREVERSECombine()
14401 EVT VT = Cond.getValueType(); in tryDemorganOfBooleanCondition() local
14405 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1); in tryDemorganOfBooleanCondition()
14417 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(0), in tryDemorganOfBooleanCondition()
14421 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(1), in tryDemorganOfBooleanCondition()
14422 DAG.getConstant(1, SDLoc(Setcc), VT), CCVal); in tryDemorganOfBooleanCondition()
14425 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, in tryDemorganOfBooleanCondition()
14426 DAG.getConstant(0, SDLoc(Setcc), VT), in tryDemorganOfBooleanCondition()
14432 return DAG.getNode(Opc, SDLoc(Cond), VT, Setcc, Xor.getOperand(0)); in tryDemorganOfBooleanCondition()
14560 EVT VT = N->getValueType(0); in tryFoldSelectIntoOp() local
14574 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel); in tryFoldSelectIntoOp()
14644 EVT VT = N->getValueType(0); in useInversedSetcc() local
14652 if (Subtarget.hasStdExtZbs() && VT.isScalarInteger() && in useInversedSetcc()
14661 return DAG.getSelect(DL, VT, in useInversedSetcc()
14696 EVT VT = N->getValueType(0); in performBUILD_VECTORCombine() local
14698 assert(!VT.isScalableVector() && "unexpected build vector"); in performBUILD_VECTORCombine()
14700 if (VT.getVectorNumElements() == 1) in performBUILD_VECTORCombine()
14707 if (!TLI.isOperationLegalOrCustom(Opcode, VT) || !TLI.isTypeLegal(VT)) in performBUILD_VECTORCombine()
14740 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps), in performBUILD_VECTORCombine()
14741 DAG.getBuildVector(VT, DL, RHSOps)); in performBUILD_VECTORCombine()
14752 EVT VT = InVec.getValueType(); in performINSERT_VECTOR_ELTCombine() local
14753 if (VT.isScalableVector()) in performINSERT_VECTOR_ELTCombine()
14778 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14780 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14782 return DAG.getNode(InVecOpcode, DL, VT, LHS, RHS); in performINSERT_VECTOR_ELTCombine()
14810 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in performINSERT_VECTOR_ELTCombine()
14821 EVT VT = N->getValueType(0); in performCONCAT_VECTORSCombine() local
14824 if (!TLI.isTypeLegal(VT)) in performCONCAT_VECTORSCombine()
14828 if (VT.isScalableVector()) in performCONCAT_VECTORSCombine()
14944 return DAG.getBitcast(VT.getSimpleVT(), StridedLoad); in performCONCAT_VECTORSCombine()
15013 EVT VT = N->getValueType(0); in combineToVWMACC() local
15016 return DAG.getNode(Opc, DL, VT, Ops); in combineToVWMACC()
15052 static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, in matchIndexAsShuffle() argument
15059 const unsigned ElementSize = VT.getScalarStoreSize(); in matchIndexAsShuffle()
15060 const unsigned NumElems = VT.getVectorNumElements(); in matchIndexAsShuffle()
15087 static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, in matchIndexAsWiderOp() argument
15096 const unsigned NumElems = VT.getVectorNumElements(); in matchIndexAsWiderOp()
15100 const unsigned ElementSize = VT.getScalarStoreSize(); in matchIndexAsWiderOp()
15230 MVT VT = N->getSimpleValueType(0); in PerformDAGCombine() local
15238 assert(Op0.getOperand(0).getValueType() == VT && in PerformDAGCombine()
15250 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0)); in PerformDAGCombine()
15252 APInt SignBit = APInt::getSignMask(FPBits).sext(VT.getSizeInBits()); in PerformDAGCombine()
15254 return DAG.getNode(ISD::XOR, DL, VT, NewFMV, in PerformDAGCombine()
15255 DAG.getConstant(SignBit, DL, VT)); in PerformDAGCombine()
15258 return DAG.getNode(ISD::AND, DL, VT, NewFMV, in PerformDAGCombine()
15259 DAG.getConstant(~SignBit, DL, VT)); in PerformDAGCombine()
15403 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15424 DAG.getNode(ISD::SRA, DL, VT, LHS, in PerformDAGCombine()
15425 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT)); in PerformDAGCombine()
15427 DAG.getNode(ISD::AND, DL, VT, SRA, in PerformDAGCombine()
15428 DAG.getConstant(TrueSImm - FalseSImm, DL, VT)); in PerformDAGCombine()
15429 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV); in PerformDAGCombine()
15443 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal); in PerformDAGCombine()
15444 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15445 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in PerformDAGCombine()
15450 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15451 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15452 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in PerformDAGCombine()
15458 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15459 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15460 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in PerformDAGCombine()
15464 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal); in PerformDAGCombine()
15465 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15466 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in PerformDAGCombine()
15477 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ); in PerformDAGCombine()
15478 return DAG.getNode(ISD::ADD, DL, VT, LHS, C); in PerformDAGCombine()
15490 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC, in PerformDAGCombine()
15492 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1)); in PerformDAGCombine()
15518 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15519 if (!VT.isVector()) in PerformDAGCombine()
15537 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT); in PerformDAGCombine()
15538 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0), in PerformDAGCombine()
15539 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound)); in PerformDAGCombine()
15543 const EVT VT = N->getValueType(0); in PerformDAGCombine() local
15566 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) { in PerformDAGCombine()
15585 SDVTList VTs = DAG.getVTList({VT, MVT::Other}); in PerformDAGCombine()
15593 Ops, VT, MGN->getMemOperand()); in PerformDAGCombine()
15599 matchIndexAsShuffle(VT, Index, MGN->getMask(), ShuffleMask)) { in PerformDAGCombine()
15600 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(), in PerformDAGCombine()
15602 MGN->getMask(), DAG.getUNDEF(VT), in PerformDAGCombine()
15606 DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask); in PerformDAGCombine()
15611 matchIndexAsWiderOp(VT, Index, MGN->getMask(), in PerformDAGCombine()
15620 unsigned ElementSize = VT.getScalarStoreSize(); in PerformDAGCombine()
15622 auto EltCnt = VT.getVectorElementCount(); in PerformDAGCombine()
15636 SDValue Result = DAG.getBitcast(VT, Gather.getValue(0)); in PerformDAGCombine()
15664 EVT VT = MSN->getValue()->getValueType(0); in PerformDAGCombine() local
15667 matchIndexAsShuffle(VT, Index, MSN->getMask(), ShuffleMask)) { in PerformDAGCombine()
15668 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(), in PerformDAGCombine()
15669 DAG.getUNDEF(VT), ShuffleMask); in PerformDAGCombine()
15734 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15735 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in PerformDAGCombine()
15737 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt, in PerformDAGCombine()
15752 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15753 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in PerformDAGCombine()
15756 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt); in PerformDAGCombine()
15889 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15891 if (!isTypeLegal(VT)) in PerformDAGCombine()
15893 if (auto Gather = matchSplatAsGather(N->getOperand(0), VT.getSimpleVT(), N, in PerformDAGCombine()
15911 const MVT VT = N->getSimpleValueType(0); in PerformDAGCombine() local
15918 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
15922 const MVT VT = N->getSimpleValueType(0); in PerformDAGCombine() local
15930 unsigned EltWidth = VT.getScalarSizeInBits(); in PerformDAGCombine()
15941 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
15954 EVT VT = N->getValueType(0); in PerformDAGCombine() local
15956 assert(SrcVT.getVectorElementType() == VT.getVectorElementType()); in PerformDAGCombine()
15958 if (SrcVT == VT) in PerformDAGCombine()
15965 const MVT VT = N->getSimpleValueType(0); in PerformDAGCombine() local
15971 const MVT M1VT = getLMUL1VT(VT); in PerformDAGCombine()
15972 if (M1VT.bitsLT(VT)) { in PerformDAGCombine()
15978 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result, in PerformDAGCombine()
15988 VT.bitsLE(getLMUL1VT(VT)) && Passthru.isUndef()) in PerformDAGCombine()
15989 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
16014 MVT VT = N->getSimpleValueType(0); in PerformDAGCombine() local
16023 const unsigned ElementSize = VT.getScalarStoreSize(); in PerformDAGCombine()
16026 return DAG.getMaskedLoad(VT, DL, Load->getChain(), Base, in PerformDAGCombine()
16062 EVT VT = N->getValueType(0); in PerformDAGCombine() local
16065 return DAG.getConstant(-1, DL, VT); in PerformDAGCombine()
16066 return DAG.getConstant(0, DL, VT); in PerformDAGCombine()
16073 EVT VT = N->getValueType(0); in PerformDAGCombine() local
16078 VT.isScalarInteger()) { in PerformDAGCombine()
16085 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in PerformDAGCombine()
16173 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
16174 if (VT.isVector()) in targetShrinkDemandedConstant()
16221 if (VT == MVT::i64) { in targetShrinkDemandedConstant()
17670 MVT ArgVT = ArgIdx.value().VT; in preAssignMask()
17689 MVT ArgVT = Ins[i].VT; in analyzeInputArgs()
17720 MVT ArgVT = Outs[i].VT; in analyzeOutputArgs()
18305 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) { in getPrefTypeAlign() argument
18307 VT.getTypeForEVT(*DAG.getContext())); in getPrefTypeAlign()
18624 MVT VT = Outs[i].VT; in CanLowerReturn() local
18627 if (RISCV::CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full, in CanLowerReturn()
19052 MVT VT) const { in getRegForInlineAsmConstraint()
19059 if (VT.isVector()) in getRegForInlineAsmConstraint()
19061 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin()) in getRegForInlineAsmConstraint()
19063 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx()) in getRegForInlineAsmConstraint()
19065 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
19069 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) in getRegForInlineAsmConstraint()
19071 if (Subtarget.hasStdExtF() && VT == MVT::f32) in getRegForInlineAsmConstraint()
19073 if (Subtarget.hasStdExtD() && VT == MVT::f64) in getRegForInlineAsmConstraint()
19082 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) in getRegForInlineAsmConstraint()
19086 if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy)) in getRegForInlineAsmConstraint()
19175 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) { in getRegForInlineAsmConstraint()
19180 if (VT == MVT::f32 || VT == MVT::Other) in getRegForInlineAsmConstraint()
19182 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) { in getRegForInlineAsmConstraint()
19226 if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy)) in getRegForInlineAsmConstraint()
19228 if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy)) in getRegForInlineAsmConstraint()
19232 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) { in getRegForInlineAsmConstraint()
19241 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
19523 EVT VT) const { in shouldConvertFpToSat()
19524 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple()) in shouldConvertFpToSat()
19609 EVT VT; in getPreIndexedAddressParts() local
19612 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19615 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19632 EVT VT; in getPostIndexedAddressParts() local
19635 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
19638 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
19655 EVT VT) const { in isFMAFasterThanFMulAndFAdd()
19656 EVT SVT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
19663 return VT.isVector() ? Subtarget.hasVInstructionsF16() in isFMAFasterThanFMulAndFAdd()
19708 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, in decomposeMulByConstant() argument
19713 if (!VT.isScalarInteger()) in decomposeMulByConstant()
19718 if (HasExtMOrZmmul && VT.getSizeInBits() > Subtarget.getXLen()) in decomposeMulByConstant()
19751 EVT VT = AddNode.getValueType(); in isMulAddWithConstProfitable() local
19752 if (VT.isVector()) in isMulAddWithConstProfitable()
19756 if (VT.getScalarSizeInBits() > Subtarget.getXLen()) in isMulAddWithConstProfitable()
19772 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, in allowsMisalignedMemoryAccesses() argument
19774 if (!VT.isVector()) { in allowsMisalignedMemoryAccesses()
19783 EVT ElemVT = VT.getVectorElementType(); in allowsMisalignedMemoryAccesses()
19943 bool RISCVTargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { in isIntDivCheap() argument
19948 return OptSize && !VT.isVector(); in isIntDivCheap()
19980 EVT VT = getValueType(DL, VTy); in isLegalInterleavedAccessType() local
19982 if (!isTypeLegal(VT)) in isLegalInterleavedAccessType()
19985 if (!isLegalElementTypeForRVV(VT.getScalarType()) || in isLegalInterleavedAccessType()
19986 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace, in isLegalInterleavedAccessType()
19990 MVT ContainerVT = VT.getSimpleVT(); in isLegalInterleavedAccessType()
20000 ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT()); in isLegalInterleavedAccessType()
20257 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT, in getRegisterByName() argument
20323 bool RISCVTargetLowering::isCtpopFast(EVT VT) const { in isCtpopFast()
20324 if (VT.isScalableVector()) in isCtpopFast()
20325 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb(); in isCtpopFast()
20326 if (VT.isFixedLengthVector() && Subtarget.hasStdExtZvbb()) in isCtpopFast()
20329 (VT == MVT::i32 || VT == MVT::i64 || VT.isFixedLengthVector()); in isCtpopFast()
20332 unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT, in getCustomCtpopCost() argument
20334 return isCtpopFast(VT) ? 0 : 1; in getCustomCtpopCost()
20373 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
20374 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit()))) in BuildSDIVPow2()
20384 EVT VT, const APInt &AndMask) const { in shouldFoldSelectWithSingleBitTest() argument
20387 return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask); in shouldFoldSelectWithSingleBitTest()