Lines Matching refs:VT
132 static inline EVT getPackedSVEVectorVT(EVT VT) { in getPackedSVEVectorVT() argument
133 switch (VT.getSimpleVT().SimpleTy) { in getPackedSVEVectorVT()
172 static inline EVT getPromotedVTForPredicate(EVT VT) { in getPromotedVTForPredicate() argument
173 assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) && in getPromotedVTForPredicate()
175 switch (VT.getVectorMinNumElements()) { in getPromotedVTForPredicate()
194 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) { in isPackedVectorType() argument
195 assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) && in isPackedVectorType()
197 return VT.isFixedLengthVector() || in isPackedVectorType()
198 VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock; in isPackedVectorType()
365 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) in AArch64TargetLowering() local
366 if (useSVEForFixedLengthVectorVT(VT)) in AArch64TargetLowering()
367 addRegisterClass(VT, &AArch64::ZPRRegClass); in AArch64TargetLowering()
369 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) in AArch64TargetLowering() local
370 if (useSVEForFixedLengthVectorVT(VT)) in AArch64TargetLowering()
371 addRegisterClass(VT, &AArch64::ZPRRegClass); in AArch64TargetLowering()
518 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in AArch64TargetLowering() local
519 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
520 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
543 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in AArch64TargetLowering() local
544 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
545 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
704 for (auto VT : {MVT::f32, MVT::f64}) in AArch64TargetLowering()
705 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal); in AArch64TargetLowering()
821 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering() local
822 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
823 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
824 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
825 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
827 for (MVT VT : MVT::integer_valuetypes()) in AArch64TargetLowering() local
828 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
986 for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32}) in AArch64TargetLowering()
987 setOperationAction(Op, VT, Custom); in AArch64TargetLowering()
1021 for (auto VT : {MVT::v1i64, MVT::v2i64}) { in AArch64TargetLowering()
1022 setOperationAction(ISD::UMAX, VT, Custom); in AArch64TargetLowering()
1023 setOperationAction(ISD::SMAX, VT, Custom); in AArch64TargetLowering()
1024 setOperationAction(ISD::UMIN, VT, Custom); in AArch64TargetLowering()
1025 setOperationAction(ISD::SMIN, VT, Custom); in AArch64TargetLowering()
1036 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, in AArch64TargetLowering()
1038 setOperationAction(ISD::SADDSAT, VT, Legal); in AArch64TargetLowering()
1039 setOperationAction(ISD::UADDSAT, VT, Legal); in AArch64TargetLowering()
1040 setOperationAction(ISD::SSUBSAT, VT, Legal); in AArch64TargetLowering()
1041 setOperationAction(ISD::USUBSAT, VT, Legal); in AArch64TargetLowering()
1044 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, in AArch64TargetLowering()
1046 setOperationAction(ISD::AVGFLOORS, VT, Legal); in AArch64TargetLowering()
1047 setOperationAction(ISD::AVGFLOORU, VT, Legal); in AArch64TargetLowering()
1048 setOperationAction(ISD::AVGCEILS, VT, Legal); in AArch64TargetLowering()
1049 setOperationAction(ISD::AVGCEILU, VT, Legal); in AArch64TargetLowering()
1050 setOperationAction(ISD::ABDS, VT, Legal); in AArch64TargetLowering()
1051 setOperationAction(ISD::ABDU, VT, Legal); in AArch64TargetLowering()
1055 for (MVT VT : { MVT::v4f16, MVT::v2f32, in AArch64TargetLowering()
1057 if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) { in AArch64TargetLowering()
1058 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in AArch64TargetLowering()
1059 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in AArch64TargetLowering()
1061 setOperationAction(ISD::VECREDUCE_FADD, VT, Legal); in AArch64TargetLowering()
1064 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, in AArch64TargetLowering()
1066 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in AArch64TargetLowering()
1067 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in AArch64TargetLowering()
1068 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in AArch64TargetLowering()
1069 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in AArch64TargetLowering()
1070 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in AArch64TargetLowering()
1078 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in AArch64TargetLowering() local
1079 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
1081 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) { in AArch64TargetLowering()
1082 setOperationAction(ISD::MULHS, VT, Legal); in AArch64TargetLowering()
1083 setOperationAction(ISD::MULHU, VT, Legal); in AArch64TargetLowering()
1085 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
1086 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
1088 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1089 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1091 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
1092 setOperationAction(ISD::CTTZ, VT, Expand); in AArch64TargetLowering()
1095 setTruncStoreAction(VT, InnerVT, Expand); in AArch64TargetLowering()
1096 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1097 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1098 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1125 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) in AArch64TargetLowering()
1126 setOperationAction(ISD::ADD, VT, Custom); in AArch64TargetLowering()
1128 for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 }) in AArch64TargetLowering()
1129 setOperationAction(ISD::FADD, VT, Custom); in AArch64TargetLowering()
1139 for (auto VT : in AArch64TargetLowering()
1141 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in AArch64TargetLowering()
1142 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in AArch64TargetLowering()
1147 for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) { in AArch64TargetLowering()
1148 setOperationAction(ISD::BITREVERSE, VT, Custom); in AArch64TargetLowering()
1149 setOperationAction(ISD::BSWAP, VT, Custom); in AArch64TargetLowering()
1150 setOperationAction(ISD::CTLZ, VT, Custom); in AArch64TargetLowering()
1151 setOperationAction(ISD::CTPOP, VT, Custom); in AArch64TargetLowering()
1152 setOperationAction(ISD::CTTZ, VT, Custom); in AArch64TargetLowering()
1153 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1154 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1155 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1156 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in AArch64TargetLowering()
1157 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in AArch64TargetLowering()
1158 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1159 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1160 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1161 setOperationAction(ISD::MUL, VT, Custom); in AArch64TargetLowering()
1162 setOperationAction(ISD::MULHS, VT, Custom); in AArch64TargetLowering()
1163 setOperationAction(ISD::MULHU, VT, Custom); in AArch64TargetLowering()
1164 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1165 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in AArch64TargetLowering()
1166 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1167 setOperationAction(ISD::SETCC, VT, Custom); in AArch64TargetLowering()
1168 setOperationAction(ISD::SDIV, VT, Custom); in AArch64TargetLowering()
1169 setOperationAction(ISD::UDIV, VT, Custom); in AArch64TargetLowering()
1170 setOperationAction(ISD::SMIN, VT, Custom); in AArch64TargetLowering()
1171 setOperationAction(ISD::UMIN, VT, Custom); in AArch64TargetLowering()
1172 setOperationAction(ISD::SMAX, VT, Custom); in AArch64TargetLowering()
1173 setOperationAction(ISD::UMAX, VT, Custom); in AArch64TargetLowering()
1174 setOperationAction(ISD::SHL, VT, Custom); in AArch64TargetLowering()
1175 setOperationAction(ISD::SRL, VT, Custom); in AArch64TargetLowering()
1176 setOperationAction(ISD::SRA, VT, Custom); in AArch64TargetLowering()
1177 setOperationAction(ISD::ABS, VT, Custom); in AArch64TargetLowering()
1178 setOperationAction(ISD::ABDS, VT, Custom); in AArch64TargetLowering()
1179 setOperationAction(ISD::ABDU, VT, Custom); in AArch64TargetLowering()
1180 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in AArch64TargetLowering()
1181 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1182 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1183 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1184 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in AArch64TargetLowering()
1185 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in AArch64TargetLowering()
1186 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in AArch64TargetLowering()
1187 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in AArch64TargetLowering()
1189 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1190 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1191 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1192 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
1193 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
1195 setOperationAction(ISD::SADDSAT, VT, Legal); in AArch64TargetLowering()
1196 setOperationAction(ISD::UADDSAT, VT, Legal); in AArch64TargetLowering()
1197 setOperationAction(ISD::SSUBSAT, VT, Legal); in AArch64TargetLowering()
1198 setOperationAction(ISD::USUBSAT, VT, Legal); in AArch64TargetLowering()
1199 setOperationAction(ISD::UREM, VT, Expand); in AArch64TargetLowering()
1200 setOperationAction(ISD::SREM, VT, Expand); in AArch64TargetLowering()
1201 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
1202 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
1206 for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) { in AArch64TargetLowering()
1207 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1208 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1212 for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32, MVT::nxv2bf16, in AArch64TargetLowering()
1214 setOperationAction(ISD::BITCAST, VT, Custom); in AArch64TargetLowering()
1216 for (auto VT : in AArch64TargetLowering()
1219 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal); in AArch64TargetLowering()
1221 for (auto VT : in AArch64TargetLowering()
1223 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1224 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1225 setOperationAction(ISD::SETCC, VT, Custom); in AArch64TargetLowering()
1226 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1227 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1228 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1229 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1231 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1232 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in AArch64TargetLowering()
1233 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1236 if (VT != MVT::nxv16i1) { in AArch64TargetLowering()
1237 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1238 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1243 for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64, in AArch64TargetLowering()
1246 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1247 setOperationAction(ISD::MSTORE, VT, Custom); in AArch64TargetLowering()
1248 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1249 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1254 for (MVT VT : MVT::scalable_vector_valuetypes()) { in AArch64TargetLowering() local
1256 setTruncStoreAction(VT, InnerVT, Expand); in AArch64TargetLowering()
1257 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1258 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1259 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1286 for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, in AArch64TargetLowering()
1288 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1289 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1290 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1291 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1292 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1293 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1294 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1295 setOperationAction(ISD::FADD, VT, Custom); in AArch64TargetLowering()
1296 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in AArch64TargetLowering()
1297 setOperationAction(ISD::FDIV, VT, Custom); in AArch64TargetLowering()
1298 setOperationAction(ISD::FMA, VT, Custom); in AArch64TargetLowering()
1299 setOperationAction(ISD::FMAXIMUM, VT, Custom); in AArch64TargetLowering()
1300 setOperationAction(ISD::FMAXNUM, VT, Custom); in AArch64TargetLowering()
1301 setOperationAction(ISD::FMINIMUM, VT, Custom); in AArch64TargetLowering()
1302 setOperationAction(ISD::FMINNUM, VT, Custom); in AArch64TargetLowering()
1303 setOperationAction(ISD::FMUL, VT, Custom); in AArch64TargetLowering()
1304 setOperationAction(ISD::FNEG, VT, Custom); in AArch64TargetLowering()
1305 setOperationAction(ISD::FSUB, VT, Custom); in AArch64TargetLowering()
1306 setOperationAction(ISD::FCEIL, VT, Custom); in AArch64TargetLowering()
1307 setOperationAction(ISD::FFLOOR, VT, Custom); in AArch64TargetLowering()
1308 setOperationAction(ISD::FNEARBYINT, VT, Custom); in AArch64TargetLowering()
1309 setOperationAction(ISD::FRINT, VT, Custom); in AArch64TargetLowering()
1310 setOperationAction(ISD::FROUND, VT, Custom); in AArch64TargetLowering()
1311 setOperationAction(ISD::FROUNDEVEN, VT, Custom); in AArch64TargetLowering()
1312 setOperationAction(ISD::FTRUNC, VT, Custom); in AArch64TargetLowering()
1313 setOperationAction(ISD::FSQRT, VT, Custom); in AArch64TargetLowering()
1314 setOperationAction(ISD::FABS, VT, Custom); in AArch64TargetLowering()
1315 setOperationAction(ISD::FP_EXTEND, VT, Custom); in AArch64TargetLowering()
1316 setOperationAction(ISD::FP_ROUND, VT, Custom); in AArch64TargetLowering()
1317 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in AArch64TargetLowering()
1318 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in AArch64TargetLowering()
1319 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in AArch64TargetLowering()
1320 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in AArch64TargetLowering()
1321 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in AArch64TargetLowering()
1323 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1324 setOperationAction(ISD::FREM, VT, Expand); in AArch64TargetLowering()
1325 setOperationAction(ISD::FPOW, VT, Expand); in AArch64TargetLowering()
1326 setOperationAction(ISD::FPOWI, VT, Expand); in AArch64TargetLowering()
1327 setOperationAction(ISD::FCOS, VT, Expand); in AArch64TargetLowering()
1328 setOperationAction(ISD::FSIN, VT, Expand); in AArch64TargetLowering()
1329 setOperationAction(ISD::FSINCOS, VT, Expand); in AArch64TargetLowering()
1330 setOperationAction(ISD::FEXP, VT, Expand); in AArch64TargetLowering()
1331 setOperationAction(ISD::FEXP2, VT, Expand); in AArch64TargetLowering()
1332 setOperationAction(ISD::FLOG, VT, Expand); in AArch64TargetLowering()
1333 setOperationAction(ISD::FLOG2, VT, Expand); in AArch64TargetLowering()
1334 setOperationAction(ISD::FLOG10, VT, Expand); in AArch64TargetLowering()
1336 setCondCodeAction(ISD::SETO, VT, Expand); in AArch64TargetLowering()
1337 setCondCodeAction(ISD::SETOLT, VT, Expand); in AArch64TargetLowering()
1338 setCondCodeAction(ISD::SETLT, VT, Expand); in AArch64TargetLowering()
1339 setCondCodeAction(ISD::SETOLE, VT, Expand); in AArch64TargetLowering()
1340 setCondCodeAction(ISD::SETLE, VT, Expand); in AArch64TargetLowering()
1341 setCondCodeAction(ISD::SETULT, VT, Expand); in AArch64TargetLowering()
1342 setCondCodeAction(ISD::SETULE, VT, Expand); in AArch64TargetLowering()
1343 setCondCodeAction(ISD::SETUGE, VT, Expand); in AArch64TargetLowering()
1344 setCondCodeAction(ISD::SETUGT, VT, Expand); in AArch64TargetLowering()
1345 setCondCodeAction(ISD::SETUEQ, VT, Expand); in AArch64TargetLowering()
1346 setCondCodeAction(ISD::SETONE, VT, Expand); in AArch64TargetLowering()
1349 for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) { in AArch64TargetLowering()
1350 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1351 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1352 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1353 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1354 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1355 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1362 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32, in AArch64TargetLowering()
1364 setOperationAction(ISD::SDIV, VT, Custom); in AArch64TargetLowering()
1365 setOperationAction(ISD::UDIV, VT, Custom); in AArch64TargetLowering()
1375 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) in AArch64TargetLowering() local
1376 if (useSVEForFixedLengthVectorVT(VT)) in AArch64TargetLowering()
1377 addTypeForFixedLengthSVE(VT); in AArch64TargetLowering()
1378 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) in AArch64TargetLowering() local
1379 if (useSVEForFixedLengthVectorVT(VT)) in AArch64TargetLowering()
1380 addTypeForFixedLengthSVE(VT); in AArch64TargetLowering()
1383 for (auto VT : {MVT::v8i8, MVT::v4i16}) in AArch64TargetLowering()
1384 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1388 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) in AArch64TargetLowering()
1389 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1390 for (auto VT : {MVT::v8f16, MVT::v4f32}) in AArch64TargetLowering()
1391 setOperationAction(ISD::FP_ROUND, VT, Custom); in AArch64TargetLowering()
1416 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, in AArch64TargetLowering()
1418 setOperationAction(ISD::BITREVERSE, VT, Custom); in AArch64TargetLowering()
1419 setOperationAction(ISD::CTTZ, VT, Custom); in AArch64TargetLowering()
1420 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1421 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1422 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1426 for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, in AArch64TargetLowering()
1428 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in AArch64TargetLowering()
1431 for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32}) in AArch64TargetLowering()
1432 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in AArch64TargetLowering()
1453 void AArch64TargetLowering::addTypeForNEON(MVT VT) { in addTypeForNEON() argument
1454 assert(VT.isVector() && "VT should be a vector type"); in addTypeForNEON()
1456 if (VT.isFloatingPoint()) { in addTypeForNEON()
1457 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT(); in addTypeForNEON()
1458 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo); in addTypeForNEON()
1459 setOperationPromotedToType(ISD::STORE, VT, PromoteTo); in addTypeForNEON()
1463 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { in addTypeForNEON()
1464 setOperationAction(ISD::FSIN, VT, Expand); in addTypeForNEON()
1465 setOperationAction(ISD::FCOS, VT, Expand); in addTypeForNEON()
1466 setOperationAction(ISD::FPOW, VT, Expand); in addTypeForNEON()
1467 setOperationAction(ISD::FLOG, VT, Expand); in addTypeForNEON()
1468 setOperationAction(ISD::FLOG2, VT, Expand); in addTypeForNEON()
1469 setOperationAction(ISD::FLOG10, VT, Expand); in addTypeForNEON()
1470 setOperationAction(ISD::FEXP, VT, Expand); in addTypeForNEON()
1471 setOperationAction(ISD::FEXP2, VT, Expand); in addTypeForNEON()
1475 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 || in addTypeForNEON()
1476 ((VT == MVT::v4f16 || VT == MVT::v8f16) && Subtarget->hasFullFP16())) in addTypeForNEON()
1477 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in addTypeForNEON()
1479 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
1480 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
1481 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
1482 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
1483 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForNEON()
1484 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
1485 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
1486 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
1487 setOperationAction(ISD::OR, VT, Custom); in addTypeForNEON()
1488 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
1489 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
1491 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
1492 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
1493 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
1495 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in addTypeForNEON()
1498 if (VT != MVT::v8i8 && VT != MVT::v16i8) in addTypeForNEON()
1499 setOperationAction(ISD::CTPOP, VT, Custom); in addTypeForNEON()
1501 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
1502 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
1503 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
1504 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
1505 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
1510 setOperationAction(Opcode, VT, Custom); in addTypeForNEON()
1512 if (!VT.isFloatingPoint()) in addTypeForNEON()
1513 setOperationAction(ISD::ABS, VT, Legal); in addTypeForNEON()
1516 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) in addTypeForNEON()
1518 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
1522 if (VT.isFloatingPoint() && in addTypeForNEON()
1523 VT.getVectorElementType() != MVT::bf16 && in addTypeForNEON()
1524 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16())) in addTypeForNEON()
1531 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
1534 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16) in addTypeForNEON()
1535 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal); in addTypeForNEON()
1536 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 64) in addTypeForNEON()
1537 setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal); in addTypeForNEON()
1550 setOperationAction(ISD::STRICT_FSETCC, VT, Expand); in addTypeForNEON()
1551 setOperationAction(ISD::STRICT_FSETCCS, VT, Expand); in addTypeForNEON()
1556 setIndexedLoadAction(im, VT, Legal); in addTypeForNEON()
1557 setIndexedStoreAction(im, VT, Legal); in addTypeForNEON()
1582 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) { in addTypeForFixedLengthSVE() argument
1583 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in addTypeForFixedLengthSVE()
1587 setOperationAction(Op, VT, Expand); in addTypeForFixedLengthSVE()
1590 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1592 if (VT.isFloatingPoint()) { in addTypeForFixedLengthSVE()
1593 setCondCodeAction(ISD::SETO, VT, Expand); in addTypeForFixedLengthSVE()
1594 setCondCodeAction(ISD::SETOLT, VT, Expand); in addTypeForFixedLengthSVE()
1595 setCondCodeAction(ISD::SETLT, VT, Expand); in addTypeForFixedLengthSVE()
1596 setCondCodeAction(ISD::SETOLE, VT, Expand); in addTypeForFixedLengthSVE()
1597 setCondCodeAction(ISD::SETLE, VT, Expand); in addTypeForFixedLengthSVE()
1598 setCondCodeAction(ISD::SETULT, VT, Expand); in addTypeForFixedLengthSVE()
1599 setCondCodeAction(ISD::SETULE, VT, Expand); in addTypeForFixedLengthSVE()
1600 setCondCodeAction(ISD::SETUGE, VT, Expand); in addTypeForFixedLengthSVE()
1601 setCondCodeAction(ISD::SETUGT, VT, Expand); in addTypeForFixedLengthSVE()
1602 setCondCodeAction(ISD::SETUEQ, VT, Expand); in addTypeForFixedLengthSVE()
1603 setCondCodeAction(ISD::SETONE, VT, Expand); in addTypeForFixedLengthSVE()
1607 if (VT.isInteger()) { in addTypeForFixedLengthSVE()
1608 MVT InnerVT = VT.changeVectorElementType(MVT::i8); in addTypeForFixedLengthSVE()
1609 while (InnerVT != VT) { in addTypeForFixedLengthSVE()
1610 setTruncStoreAction(VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1611 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1612 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1620 if (VT.isFloatingPoint()) { in addTypeForFixedLengthSVE()
1621 MVT InnerVT = VT.changeVectorElementType(MVT::f16); in addTypeForFixedLengthSVE()
1622 while (InnerVT != VT) { in addTypeForFixedLengthSVE()
1623 setTruncStoreAction(VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1624 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1631 setOperationAction(ISD::ABS, VT, Custom); in addTypeForFixedLengthSVE()
1632 setOperationAction(ISD::ADD, VT, Custom); in addTypeForFixedLengthSVE()
1633 setOperationAction(ISD::AND, VT, Custom); in addTypeForFixedLengthSVE()
1634 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1635 setOperationAction(ISD::BITCAST, VT, Custom); in addTypeForFixedLengthSVE()
1636 setOperationAction(ISD::BITREVERSE, VT, Custom); in addTypeForFixedLengthSVE()
1637 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1638 setOperationAction(ISD::BSWAP, VT, Custom); in addTypeForFixedLengthSVE()
1639 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in addTypeForFixedLengthSVE()
1640 setOperationAction(ISD::CTLZ, VT, Custom); in addTypeForFixedLengthSVE()
1641 setOperationAction(ISD::CTPOP, VT, Custom); in addTypeForFixedLengthSVE()
1642 setOperationAction(ISD::CTTZ, VT, Custom); in addTypeForFixedLengthSVE()
1643 setOperationAction(ISD::FABS, VT, Custom); in addTypeForFixedLengthSVE()
1644 setOperationAction(ISD::FADD, VT, Custom); in addTypeForFixedLengthSVE()
1645 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForFixedLengthSVE()
1646 setOperationAction(ISD::FCEIL, VT, Custom); in addTypeForFixedLengthSVE()
1647 setOperationAction(ISD::FDIV, VT, Custom); in addTypeForFixedLengthSVE()
1648 setOperationAction(ISD::FFLOOR, VT, Custom); in addTypeForFixedLengthSVE()
1649 setOperationAction(ISD::FMA, VT, Custom); in addTypeForFixedLengthSVE()
1650 setOperationAction(ISD::FMAXIMUM, VT, Custom); in addTypeForFixedLengthSVE()
1651 setOperationAction(ISD::FMAXNUM, VT, Custom); in addTypeForFixedLengthSVE()
1652 setOperationAction(ISD::FMINIMUM, VT, Custom); in addTypeForFixedLengthSVE()
1653 setOperationAction(ISD::FMINNUM, VT, Custom); in addTypeForFixedLengthSVE()
1654 setOperationAction(ISD::FMUL, VT, Custom); in addTypeForFixedLengthSVE()
1655 setOperationAction(ISD::FNEARBYINT, VT, Custom); in addTypeForFixedLengthSVE()
1656 setOperationAction(ISD::FNEG, VT, Custom); in addTypeForFixedLengthSVE()
1657 setOperationAction(ISD::FP_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1658 setOperationAction(ISD::FP_ROUND, VT, Custom); in addTypeForFixedLengthSVE()
1659 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForFixedLengthSVE()
1660 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForFixedLengthSVE()
1661 setOperationAction(ISD::FRINT, VT, Custom); in addTypeForFixedLengthSVE()
1662 setOperationAction(ISD::FROUND, VT, Custom); in addTypeForFixedLengthSVE()
1663 setOperationAction(ISD::FROUNDEVEN, VT, Custom); in addTypeForFixedLengthSVE()
1664 setOperationAction(ISD::FSQRT, VT, Custom); in addTypeForFixedLengthSVE()
1665 setOperationAction(ISD::FSUB, VT, Custom); in addTypeForFixedLengthSVE()
1666 setOperationAction(ISD::FTRUNC, VT, Custom); in addTypeForFixedLengthSVE()
1667 setOperationAction(ISD::LOAD, VT, Custom); in addTypeForFixedLengthSVE()
1668 setOperationAction(ISD::MGATHER, VT, Custom); in addTypeForFixedLengthSVE()
1669 setOperationAction(ISD::MLOAD, VT, Custom); in addTypeForFixedLengthSVE()
1670 setOperationAction(ISD::MSCATTER, VT, Custom); in addTypeForFixedLengthSVE()
1671 setOperationAction(ISD::MSTORE, VT, Custom); in addTypeForFixedLengthSVE()
1672 setOperationAction(ISD::MUL, VT, Custom); in addTypeForFixedLengthSVE()
1673 setOperationAction(ISD::MULHS, VT, Custom); in addTypeForFixedLengthSVE()
1674 setOperationAction(ISD::MULHU, VT, Custom); in addTypeForFixedLengthSVE()
1675 setOperationAction(ISD::OR, VT, Custom); in addTypeForFixedLengthSVE()
1676 setOperationAction(ISD::SDIV, VT, Custom); in addTypeForFixedLengthSVE()
1677 setOperationAction(ISD::SELECT, VT, Custom); in addTypeForFixedLengthSVE()
1678 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForFixedLengthSVE()
1679 setOperationAction(ISD::SHL, VT, Custom); in addTypeForFixedLengthSVE()
1680 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1681 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom); in addTypeForFixedLengthSVE()
1682 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in addTypeForFixedLengthSVE()
1683 setOperationAction(ISD::SMAX, VT, Custom); in addTypeForFixedLengthSVE()
1684 setOperationAction(ISD::SMIN, VT, Custom); in addTypeForFixedLengthSVE()
1685 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1686 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in addTypeForFixedLengthSVE()
1687 setOperationAction(ISD::SRA, VT, Custom); in addTypeForFixedLengthSVE()
1688 setOperationAction(ISD::SRL, VT, Custom); in addTypeForFixedLengthSVE()
1689 setOperationAction(ISD::STORE, VT, Custom); in addTypeForFixedLengthSVE()
1690 setOperationAction(ISD::SUB, VT, Custom); in addTypeForFixedLengthSVE()
1691 setOperationAction(ISD::TRUNCATE, VT, Custom); in addTypeForFixedLengthSVE()
1692 setOperationAction(ISD::UDIV, VT, Custom); in addTypeForFixedLengthSVE()
1693 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in addTypeForFixedLengthSVE()
1694 setOperationAction(ISD::UMAX, VT, Custom); in addTypeForFixedLengthSVE()
1695 setOperationAction(ISD::UMIN, VT, Custom); in addTypeForFixedLengthSVE()
1696 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in addTypeForFixedLengthSVE()
1697 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in addTypeForFixedLengthSVE()
1698 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in addTypeForFixedLengthSVE()
1699 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in addTypeForFixedLengthSVE()
1700 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in addTypeForFixedLengthSVE()
1701 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in addTypeForFixedLengthSVE()
1702 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in addTypeForFixedLengthSVE()
1703 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForFixedLengthSVE()
1704 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in addTypeForFixedLengthSVE()
1705 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in addTypeForFixedLengthSVE()
1706 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in addTypeForFixedLengthSVE()
1707 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in addTypeForFixedLengthSVE()
1708 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in addTypeForFixedLengthSVE()
1709 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForFixedLengthSVE()
1710 setOperationAction(ISD::VSELECT, VT, Custom); in addTypeForFixedLengthSVE()
1711 setOperationAction(ISD::XOR, VT, Custom); in addTypeForFixedLengthSVE()
1712 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1715 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON() argument
1716 addRegisterClass(VT, &AArch64::FPR64RegClass); in addDRTypeForNEON()
1717 addTypeForNEON(VT); in addDRTypeForNEON()
1720 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON() argument
1721 addRegisterClass(VT, &AArch64::FPR128RegClass); in addQRTypeForNEON()
1722 addTypeForNEON(VT); in addQRTypeForNEON()
1726 LLVMContext &C, EVT VT) const { in getSetCCResultType()
1727 if (!VT.isVector()) in getSetCCResultType()
1729 if (VT.isScalableVector()) in getSetCCResultType()
1730 return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount()); in getSetCCResultType()
1731 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
1808 EVT VT = Op.getValueType(); in optimizeLogicalImm() local
1815 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0), in optimizeLogicalImm()
1816 TLO.DAG.getConstant(NewImm, DL, VT)); in optimizeLogicalImm()
1821 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT); in optimizeLogicalImm()
1823 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0); in optimizeLogicalImm()
1839 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
1840 if (VT.isVector()) in targetShrinkDemandedConstant()
1843 unsigned Size = VT.getSizeInBits(); in targetShrinkDemandedConstant()
1940 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode() local
1941 unsigned MemBits = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
1960 MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); in computeKnownBitsForTargetNode() local
1962 if (VT == MVT::v8i8 || VT == MVT::v16i8) { in computeKnownBitsForTargetNode()
1966 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { in computeKnownBitsForTargetNode()
1984 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, in allowsMisalignedMemoryAccesses() argument
1991 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 || in allowsMisalignedMemoryAccesses()
2002 VT == MVT::v2i64; in allowsMisalignedMemoryAccesses()
2661 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT);
2662 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
2663 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
2667 EVT VT);
2867 EVT VT = LHS.getValueType(); in emitStrictFPComparison() local
2868 assert(VT != MVT::f128); in emitStrictFPComparison()
2872 if (VT == MVT::f16 && !FullFP16) { in emitStrictFPComparison()
2878 VT = MVT::f32; in emitStrictFPComparison()
2882 return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS}); in emitStrictFPComparison()
2887 EVT VT = LHS.getValueType(); in emitComparison() local
2890 if (VT.isFloatingPoint()) { in emitComparison()
2891 assert(VT != MVT::f128); in emitComparison()
2892 if (VT == MVT::f16 && !FullFP16) { in emitComparison()
2895 VT = MVT::f32; in emitComparison()
2897 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); in emitComparison()
2921 DAG.getVTList(VT, MVT_CC), in emitComparison()
2933 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS) in emitComparison()
3251 EVT VT = Op.getValueType(); in getCmpOperandFoldingProfit() local
3252 if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63)) in getCmpOperandFoldingProfit()
3263 EVT VT = RHS.getValueType(); in getAArch64Cmp() local
3272 if ((VT == MVT::i32 && C != 0x80000000 && in getAArch64Cmp()
3274 (VT == MVT::i64 && C != 0x80000000ULL && in getAArch64Cmp()
3277 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
3278 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
3283 if ((VT == MVT::i32 && C != 0 && in getAArch64Cmp()
3285 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { in getAArch64Cmp()
3287 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
3288 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
3293 if ((VT == MVT::i32 && C != INT32_MAX && in getAArch64Cmp()
3295 (VT == MVT::i64 && C != INT64_MAX && in getAArch64Cmp()
3298 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
3299 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
3304 if ((VT == MVT::i32 && C != UINT32_MAX && in getAArch64Cmp()
3306 (VT == MVT::i64 && C != UINT64_MAX && in getAArch64Cmp()
3309 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
3310 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
3568 EVT VT = Value.getValueType(); in valueToCarryFlag() local
3569 SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value; in valueToCarryFlag()
3570 SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT); in valueToCarryFlag()
3572 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::Glue), Op0, Op1); in valueToCarryFlag()
3578 static SDValue carryFlagToValue(SDValue Flag, EVT VT, SelectionDAG &DAG, in carryFlagToValue() argument
3582 SDValue Zero = DAG.getConstant(0, DL, VT); in carryFlagToValue()
3583 SDValue One = DAG.getConstant(1, DL, VT); in carryFlagToValue()
3586 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag); in carryFlagToValue()
3590 static SDValue overflowFlagToValue(SDValue Flag, EVT VT, SelectionDAG &DAG) { in overflowFlagToValue() argument
3593 SDValue Zero = DAG.getConstant(0, DL, VT); in overflowFlagToValue()
3594 SDValue One = DAG.getConstant(1, DL, VT); in overflowFlagToValue()
3596 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag); in overflowFlagToValue()
3686 EVT VT = Op.getValueType(); in LowerFP_EXTEND() local
3687 if (VT.isScalableVector()) in LowerFP_EXTEND()
3690 if (useSVEForFixedLengthVectorVT(VT)) in LowerFP_EXTEND()
3728 EVT VT = Op.getValueType(); in LowerVectorFP_TO_INT() local
3730 if (VT.isScalableVector()) { in LowerVectorFP_TO_INT()
3737 if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT)) in LowerVectorFP_TO_INT()
3750 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerVectorFP_TO_INT()
3758 uint64_t VTSize = VT.getFixedSizeInBits(); in LowerVectorFP_TO_INT()
3766 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3772 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3778 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), in LowerVectorFP_TO_INT()
3779 VT.getVectorNumElements()); in LowerVectorFP_TO_INT()
3783 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerVectorFP_TO_INT()
3787 return DAG.getNode(Op.getOpcode(), dl, VT, Ext); in LowerVectorFP_TO_INT()
3797 EVT ScalarVT = VT.getScalarType(); in LowerVectorFP_TO_INT()
3970 EVT VT = Op.getValueType(); in LowerVectorINT_TO_FP() local
3977 if (VT.isScalableVector()) { in LowerVectorINT_TO_FP()
3983 return DAG.getNode(Opc, dl, VT, In); in LowerVectorINT_TO_FP()
3991 if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT)) in LowerVectorINT_TO_FP()
3994 uint64_t VTSize = VT.getFixedSizeInBits(); in LowerVectorINT_TO_FP()
4004 ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other}, in LowerVectorINT_TO_FP()
4008 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
4013 EVT CastVT = VT.changeVectorElementTypeToInteger(); in LowerVectorINT_TO_FP()
4016 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op.getOperand(0), In}); in LowerVectorINT_TO_FP()
4017 return DAG.getNode(Opc, dl, VT, In); in LowerVectorINT_TO_FP()
4022 if (VT.getVectorNumElements() == 1) { in LowerVectorINT_TO_FP()
4026 EVT ScalarVT = VT.getScalarType(); in LowerVectorINT_TO_FP()
4189 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR() local
4196 unsigned EltSize = VT.getScalarSizeInBits(); in isExtendedBUILD_VECTOR()
4222 EVT VT = N->getValueType(0); in skipExtensionForVectorMULL() local
4224 unsigned EltSize = VT.getScalarSizeInBits() / 2; in skipExtensionForVectorMULL()
4225 unsigned NumElts = VT.getVectorNumElements(); in skipExtensionForVectorMULL()
4340 EVT VT = Op.getValueType(); in LowerMUL() local
4343 bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64; in LowerMUL()
4345 if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON)) in LowerMUL()
4350 assert(VT.is128BitVector() && VT.isInteger() && in LowerMUL()
4382 if (VT == MVT::v2i64) in LowerMUL()
4400 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
4408 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
4409 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
4411 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
4415 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT, in getPTrue() argument
4417 if (VT == MVT::nxv1i1 && Pattern == AArch64SVEPredPattern::all) in getPTrue()
4419 return DAG.getNode(AArch64ISD::PTRUE, DL, VT, in getPTrue()
4425 static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) { in getSVEPredicateBitCast() argument
4430 VT.getVectorElementType() == MVT::i1 && in getSVEPredicateBitCast()
4432 assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) && in getSVEPredicateBitCast()
4439 if (InVT == VT) in getSVEPredicateBitCast()
4442 SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op); in getSVEPredicateBitCast()
4448 if (InVT.bitsGT(VT)) in getSVEPredicateBitCast()
4458 Mask = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Mask); in getSVEPredicateBitCast()
4459 return DAG.getNode(ISD::AND, DL, VT, Reinterpret, Mask); in getSVEPredicateBitCast()
4844 bool AArch64TargetLowering::shouldExtendGSIndex(EVT VT, EVT &EltTy) const { in shouldExtendGSIndex() argument
4845 if (VT.getVectorElementType() == MVT::i8 || in shouldExtendGSIndex()
4846 VT.getVectorElementType() == MVT::i16) { in shouldExtendGSIndex()
4931 EVT VT = Op.getValueType(); in LowerMGATHER() local
4939 SDValue Ops[] = {Chain, DAG.getUNDEF(VT), Mask, BasePtr, Index, Scale}; in LowerMGATHER()
4943 SDValue Select = DAG.getSelect(DL, VT, Mask, Load, PassThru); in LowerMGATHER()
4966 if (VT.isFixedLengthVector()) { in LowerMGATHER()
4971 EVT DataVT = VT.changeVectorElementTypeToInteger(); in LowerMGATHER()
4975 EVT PromotedVT = VT.changeVectorElementType(MVT::i32); in LowerMGATHER()
4979 PromotedVT = VT.changeVectorElementType(MVT::i64); in LowerMGATHER()
5009 if (VT.isFloatingPoint()) in LowerMGATHER()
5010 Result = DAG.getNode(ISD::BITCAST, DL, VT, Result); in LowerMGATHER()
5030 EVT VT = StoreVal.getValueType(); in LowerMSCATTER() local
5054 if (VT.isFixedLengthVector()) { in LowerMSCATTER()
5059 if (VT.isFloatingPoint()) { in LowerMSCATTER()
5060 VT = VT.changeVectorElementTypeToInteger(); in LowerMSCATTER()
5062 StoreVal = DAG.getNode(ISD::BITCAST, DL, VT, StoreVal); in LowerMSCATTER()
5066 EVT PromotedVT = VT.changeVectorElementType(MVT::i32); in LowerMSCATTER()
5067 if (VT.getVectorElementType() == MVT::i64 || in LowerMSCATTER()
5070 PromotedVT = VT.changeVectorElementType(MVT::i64); in LowerMSCATTER()
5079 if (PromotedVT != VT) in LowerMSCATTER()
5104 EVT VT = Op->getValueType(0); in LowerMLOAD() local
5107 VT, in LowerMLOAD()
5118 VT, DL, LoadNode->getChain(), LoadNode->getBasePtr(), in LowerMLOAD()
5119 LoadNode->getOffset(), Mask, DAG.getUNDEF(VT), LoadNode->getMemoryVT(), in LowerMLOAD()
5123 SDValue Result = DAG.getSelect(DL, VT, Mask, Load, PassThru); in LowerMLOAD()
5130 EVT VT, EVT MemVT, in LowerTruncateVectorStore() argument
5132 assert(VT.isVector() && "VT should be a vector type"); in LowerTruncateVectorStore()
5133 assert(MemVT == MVT::v4i8 && VT == MVT::v4i16); in LowerTruncateVectorStore()
5171 EVT VT = Value.getValueType(); in LowerSTORE() local
5174 if (VT.isVector()) { in LowerSTORE()
5176 VT, in LowerSTORE()
5189 if (StoreNode->isTruncatingStore() && VT == MVT::v4i16 && in LowerSTORE()
5191 return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG); in LowerSTORE()
5290 EVT VT = Op->getValueType(0); in LowerLOAD() local
5291 assert((VT == MVT::v4i16 || VT == MVT::v4i32) && "Expected v4i16 or v4i32"); in LowerLOAD()
5313 if (VT == MVT::v4i32) in LowerLOAD()
5320 MVT VT = Op.getSimpleValueType(); in LowerABS() local
5322 if (VT.isVector()) in LowerABS()
5326 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerABS()
5330 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
5331 Op.getOperand(0), DAG.getConstant(0, DL, VT)); in LowerABS()
5332 return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg, in LowerABS()
5628 bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const { in mergeStoresAfterLegalization()
5633 EVT VT, bool OverrideNEON) const { in useSVEForFixedLengthVectorVT() argument
5634 if (!VT.isFixedLengthVector() || !VT.isSimple()) in useSVEForFixedLengthVectorVT()
5638 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) { in useSVEForFixedLengthVectorVT()
5655 if (OverrideNEON && (VT.is128BitVector() || VT.is64BitVector())) in useSVEForFixedLengthVectorVT()
5659 if (VT.getFixedSizeInBits() <= 128) in useSVEForFixedLengthVectorVT()
5667 if (VT.getFixedSizeInBits() > Subtarget->getMinSVEVectorSizeInBits()) in useSVEForFixedLengthVectorVT()
5672 if (!VT.isPow2VectorType()) in useSVEForFixedLengthVectorVT()
5766 if (any_of(Outs, [](ISD::OutputArg &Out){ return Out.VT.isScalableVector(); })) in LowerFormalArguments()
5784 MVT ValVT = Ins[i].VT; in LowerFormalArguments()
6247 MVT ArgVT = Outs[i].VT; in analyzeCallOperands()
6512 if (!Outs[i].IsFixed && Outs[i].VT.isScalableVector()) in LowerCall()
6615 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT); in LowerCall()
6727 Outs[0].VT == MVT::i64) { in LowerCall()
6730 assert(!Ins.empty() && Ins[0].VT == MVT::i64 && in LowerCall()
7724 EVT VT = Op.getValueType(); in LowerFCOPYSIGN() local
7725 EVT IntVT = VT.changeTypeToInteger(); in LowerFCOPYSIGN()
7732 if (SrcVT.bitsLT(VT)) in LowerFCOPYSIGN()
7733 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
7734 else if (SrcVT.bitsGT(VT)) in LowerFCOPYSIGN()
7735 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
7737 if (VT.isScalableVector()) in LowerFCOPYSIGN()
7739 getPackedSVEVectorVT(VT.getVectorElementType().changeTypeToInteger()); in LowerFCOPYSIGN()
7741 if (VT != In2.getValueType()) in LowerFCOPYSIGN()
7744 auto BitCast = [this](EVT VT, SDValue Op, SelectionDAG &DAG) { in LowerFCOPYSIGN() argument
7745 if (VT.isScalableVector()) in LowerFCOPYSIGN()
7746 return getSVESafeBitCast(VT, Op, DAG); in LowerFCOPYSIGN()
7748 return DAG.getBitcast(VT, Op); in LowerFCOPYSIGN()
7754 if (!VT.isVector()) { in LowerFCOPYSIGN()
7764 if (VT.isVector()) { in LowerFCOPYSIGN()
7767 } else if (VT == MVT::f64) { in LowerFCOPYSIGN()
7770 } else if (VT == MVT::f32) { in LowerFCOPYSIGN()
7773 } else if (VT == MVT::f16) { in LowerFCOPYSIGN()
7786 if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
7795 if (VT == MVT::f16) in LowerFCOPYSIGN()
7796 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, BSP); in LowerFCOPYSIGN()
7797 if (VT == MVT::f32) in LowerFCOPYSIGN()
7798 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, BSP); in LowerFCOPYSIGN()
7799 if (VT == MVT::f64) in LowerFCOPYSIGN()
7800 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, BSP); in LowerFCOPYSIGN()
7802 return BitCast(VT, BSP, DAG); in LowerFCOPYSIGN()
7826 EVT VT = Op.getValueType(); in LowerCTPOP_PARITY() local
7828 if (VT == MVT::i32 || VT == MVT::i64) { in LowerCTPOP_PARITY()
7829 if (VT == MVT::i32) in LowerCTPOP_PARITY()
7842 if (VT == MVT::i64) in LowerCTPOP_PARITY()
7845 } else if (VT == MVT::i128) { in LowerCTPOP_PARITY()
7862 if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) in LowerCTPOP_PARITY()
7865 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || in LowerCTPOP_PARITY()
7866 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && in LowerCTPOP_PARITY()
7869 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; in LowerCTPOP_PARITY()
7875 unsigned NumElts = VT.is64BitVector() ? 8 : 16; in LowerCTPOP_PARITY()
7876 while (EltSize != VT.getScalarSizeInBits()) { in LowerCTPOP_PARITY()
7889 EVT VT = Op.getValueType(); in LowerCTTZ() local
7890 assert(VT.isScalableVector() || in LowerCTTZ()
7892 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors())); in LowerCTTZ()
7895 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0)); in LowerCTTZ()
7896 return DAG.getNode(ISD::CTLZ, DL, VT, RBIT); in LowerCTTZ()
7902 EVT VT = Op.getValueType(); in LowerMinMax() local
7923 if (VT.isScalableVector() || in LowerMinMax()
7925 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors())) { in LowerMinMax()
7942 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC); in LowerMinMax()
7943 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in LowerMinMax()
7948 EVT VT = Op.getValueType(); in LowerBitreverse() local
7950 if (VT.isScalableVector() || in LowerBitreverse()
7952 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors())) in LowerBitreverse()
7959 switch (VT.getSimpleVT().SimpleTy) { in LowerBitreverse()
7992 return DAG.getNode(AArch64ISD::NVCAST, DL, VT, in LowerBitreverse()
8013 EVT VT = Op.getValueType(); in LowerSETCC() local
8014 SDValue TVal = DAG.getConstant(1, dl, VT); in LowerSETCC()
8015 SDValue FVal = DAG.getConstant(0, dl, VT); in LowerSETCC()
8039 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); in LowerSETCC()
8066 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp); in LowerSETCC()
8076 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSETCC()
8079 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSETCC()
8121 EVT VT = LHS.getValueType(); in LowerSELECT_CC() local
8123 DAG.getNode(ISD::SRA, dl, VT, LHS, in LowerSELECT_CC()
8124 DAG.getConstant(VT.getSizeInBits() - 1, dl, VT)); in LowerSELECT_CC()
8125 return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT)); in LowerSELECT_CC()
8237 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
8238 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); in LowerSELECT_CC()
8245 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
8273 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSELECT_CC()
8279 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSELECT_CC()
8601 EVT VT = Op.getValueType(); in LowerVAARG() local
8614 if (VT.isScalableVector()) in LowerVAARG()
8625 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); in LowerVAARG()
8632 if (VT.isInteger() && !VT.isVector()) in LowerVAARG()
8635 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { in LowerVAARG()
8655 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
8662 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo()); in LowerVAARG()
8670 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
8676 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
8681 DAG.getValueType(VT)); in LowerFRAMEADDR()
8690 EVT VT = getPointerTy(DAG.getDataLayout()); in LowerSPONENTRY() local
8693 return DAG.getFrameIndex(FI, VT); in LowerSPONENTRY()
8702 getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const { in getRegisterByName() argument
8720 EVT VT = Op.getValueType(); in LowerADDROFRETURNADDR() local
8724 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT); in LowerADDROFRETURNADDR()
8727 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset); in LowerADDROFRETURNADDR()
8736 EVT VT = Op.getValueType(); in LowerRETURNADDR() local
8744 VT, DL, DAG.getEntryNode(), in LowerRETURNADDR()
8745 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo()); in LowerRETURNADDR()
8750 ReturnAddress = DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); in LowerRETURNADDR()
8759 St = DAG.getMachineNode(AArch64::XPACI, DL, VT, ReturnAddress); in LowerRETURNADDR()
8764 St = DAG.getMachineNode(AArch64::XPACLRI, DL, VT, Chain); in LowerRETURNADDR()
8785 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
8792 if (VT == MVT::f64) in isFPImmLegal()
8794 else if (VT == MVT::f32) in isFPImmLegal()
8796 else if (VT == MVT::f16 && Subtarget->hasFullFP16()) in isFPImmLegal()
8804 if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) { in isFPImmLegal()
8811 AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(), in isFPImmLegal()
8817 LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString() in isFPImmLegal()
8829 EVT VT = Operand.getValueType(); in getEstimate() local
8831 (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 || in getEstimate()
8832 VT == MVT::f32 || VT == MVT::v1f32 || VT == MVT::v2f32 || in getEstimate()
8833 VT == MVT::v4f32)) || in getEstimate()
8835 (VT == MVT::nxv8f16 || VT == MVT::nxv4f32 || VT == MVT::nxv2f64))) { in getEstimate()
8842 ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2; in getEstimate()
8844 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); in getEstimate()
8854 EVT VT = Op.getValueType(); in getSqrtInputTest() local
8855 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSqrtInputTest()
8856 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in getSqrtInputTest()
8876 EVT VT = Operand.getValueType(); in getSqrtEstimate() local
8884 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate, in getSqrtEstimate()
8886 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags); in getSqrtEstimate()
8887 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getSqrtEstimate()
8890 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags); in getSqrtEstimate()
8906 EVT VT = Operand.getValueType(); in getRecipEstimate() local
8914 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand, in getRecipEstimate()
8916 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getRecipEstimate()
9062 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
9066 if (VT.isScalableVector()) in getRegForInlineAsmConstraint()
9068 if (Subtarget->hasLS64() && VT.getSizeInBits() == 512) in getRegForInlineAsmConstraint()
9070 if (VT.getFixedSizeInBits() == 64) in getRegForInlineAsmConstraint()
9076 if (VT.isScalableVector()) { in getRegForInlineAsmConstraint()
9077 if (VT.getVectorElementType() != MVT::i1) in getRegForInlineAsmConstraint()
9081 uint64_t VTSize = VT.getFixedSizeInBits(); in getRegForInlineAsmConstraint()
9097 if (VT.isScalableVector()) in getRegForInlineAsmConstraint()
9099 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
9105 if (VT.isScalableVector()) in getRegForInlineAsmConstraint()
9112 if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1) in getRegForInlineAsmConstraint()
9125 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
9138 if (VT != MVT::Other && VT.getSizeInBits() == 64) { in getRegForInlineAsmConstraint()
9323 EVT VT = V64Reg.getValueType(); in WidenVector() local
9324 unsigned NarrowSize = VT.getVectorNumElements(); in WidenVector()
9325 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in WidenVector()
9343 EVT VT = V128Reg.getValueType(); in NarrowVector() local
9344 unsigned WideSize = VT.getVectorNumElements(); in NarrowVector()
9345 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
9359 EVT VT = Op.getValueType(); in ReconstructShuffle() local
9360 assert(!VT.isScalableVector() && in ReconstructShuffle()
9362 unsigned NumElts = VT.getVectorNumElements(); in ReconstructShuffle()
9422 unsigned OutputFactor = VT.getScalarSizeInBits() / 8; in ReconstructShuffle()
9475 return DAG.getBitcast(VT, Shuffle); in ReconstructShuffle()
9487 EVT SmallestEltTy = VT.getVectorElementType(); in ReconstructShuffle()
9495 VT.getScalarSizeInBits() / SmallestEltTy.getFixedSizeInBits(); in ReconstructShuffle()
9496 uint64_t VTSize = VT.getFixedSizeInBits(); in ReconstructShuffle()
9595 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { in ReconstructShuffle()
9608 VT.getScalarSizeInBits()); in ReconstructShuffle()
9633 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
9643 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { in isSingletonEXTMask() argument
9644 unsigned NumElts = VT.getVectorNumElements(); in isSingletonEXTMask()
9726 static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize, in isWideDUPMask() argument
9731 if (BlockSize <= VT.getScalarSizeInBits()) in isWideDUPMask()
9733 if (BlockSize % VT.getScalarSizeInBits() != 0) in isWideDUPMask()
9735 if (VT.getSizeInBits() % BlockSize != 0) in isWideDUPMask()
9738 size_t SingleVecNumElements = VT.getVectorNumElements(); in isWideDUPMask()
9739 size_t NumEltsPerBlock = BlockSize / VT.getScalarSizeInBits(); in isWideDUPMask()
9740 size_t NumBlocks = VT.getSizeInBits() / BlockSize; in isWideDUPMask()
9799 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT, in isEXTMask() argument
9805 unsigned NumElts = VT.getVectorNumElements(); in isEXTMask()
9840 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { in isREVMask() argument
9844 unsigned EltSz = VT.getScalarSizeInBits(); in isREVMask()
9848 unsigned NumElts = VT.getVectorNumElements(); in isREVMask()
9867 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIPMask() argument
9868 unsigned NumElts = VT.getVectorNumElements(); in isZIPMask()
9883 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZPMask() argument
9884 unsigned NumElts = VT.getVectorNumElements(); in isUZPMask()
9896 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRNMask() argument
9897 unsigned NumElts = VT.getVectorNumElements(); in isTRNMask()
9912 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIP_v_undef_Mask() argument
9913 unsigned NumElts = VT.getVectorNumElements(); in isZIP_v_undef_Mask()
9931 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZP_v_undef_Mask() argument
9932 unsigned Half = VT.getVectorNumElements() / 2; in isUZP_v_undef_Mask()
9950 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRN_v_undef_Mask() argument
9951 unsigned NumElts = VT.getVectorNumElements(); in isTRN_v_undef_Mask()
10002 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) { in isConcatMask() argument
10003 if (VT.getSizeInBits() != 128) in isConcatMask()
10006 unsigned NumElts = VT.getVectorNumElements(); in isConcatMask()
10024 EVT VT = Op.getValueType(); in tryFormConcatFromShuffle() local
10029 if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() || in tryFormConcatFromShuffle()
10030 VT.getVectorElementType() != V1.getValueType().getVectorElementType()) in tryFormConcatFromShuffle()
10035 if (!isConcatMask(Mask, VT, SplitV0)) in tryFormConcatFromShuffle()
10038 EVT CastVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in tryFormConcatFromShuffle()
10047 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
10106 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
10120 if (VT.getScalarSizeInBits() == 16) { in GeneratePerfectShuffle()
10124 assert(VT.getScalarSizeInBits() == 32 && in GeneratePerfectShuffle()
10135 if (VT == MVT::v4i16) { in GeneratePerfectShuffle()
10146 return DAG.getBitcast(VT, Ins); in GeneratePerfectShuffle()
10154 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
10161 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
10162 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
10163 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
10165 if (VT.getVectorElementType() == MVT::i16 || in GeneratePerfectShuffle()
10166 VT.getVectorElementType() == MVT::f16 || in GeneratePerfectShuffle()
10167 VT.getVectorElementType() == MVT::bf16) in GeneratePerfectShuffle()
10168 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
10170 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
10171 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
10176 EVT EltTy = VT.getVectorElementType(); in GeneratePerfectShuffle()
10189 if (VT.getSizeInBits() == 64) in GeneratePerfectShuffle()
10192 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane); in GeneratePerfectShuffle()
10198 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS, in GeneratePerfectShuffle()
10202 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10205 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10208 return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10211 return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10214 return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10217 return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10311 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT, in constructDup() argument
10360 unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2; in constructDup()
10361 Lane -= Idx * VT.getVectorNumElements() / 2; in constructDup()
10363 } else if (VT.getSizeInBits() == 64) { in constructDup()
10367 return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64)); in constructDup()
10373 static bool isWideTypeMask(ArrayRef<int> M, EVT VT, in isWideTypeMask() argument
10375 unsigned NumElts = VT.getVectorNumElements(); in isWideTypeMask()
10420 EVT VT = Op.getValueType(); in tryWidenMaskForShuffle() local
10421 EVT ScalarVT = VT.getVectorElementType(); in tryWidenMaskForShuffle()
10434 if (isWideTypeMask(Mask, VT, NewMask)) { in tryWidenMaskForShuffle()
10435 MVT NewEltVT = VT.isFloatingPoint() in tryWidenMaskForShuffle()
10438 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2); in tryWidenMaskForShuffle()
10442 return DAG.getBitcast(VT, in tryWidenMaskForShuffle()
10453 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE() local
10457 if (useSVEForFixedLengthVectorVT(VT)) in LowerVECTOR_SHUFFLE()
10469 assert(V1.getValueType() == VT && "Unexpected VECTOR_SHUFFLE type!"); in LowerVECTOR_SHUFFLE()
10470 assert(ShuffleMask.size() == VT.getVectorNumElements() && in LowerVECTOR_SHUFFLE()
10486 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane)); in LowerVECTOR_SHUFFLE()
10490 return constructDup(V1, Lane, dl, VT, Opcode, DAG); in LowerVECTOR_SHUFFLE()
10496 if (isWideDUPMask(ShuffleMask, VT, LaneSize, Lane)) { in LowerVECTOR_SHUFFLE()
10502 unsigned NewEltCount = VT.getSizeInBits() / LaneSize; in LowerVECTOR_SHUFFLE()
10508 return DAG.getBitcast(VT, V1); in LowerVECTOR_SHUFFLE()
10512 if (isREVMask(ShuffleMask, VT, 64)) in LowerVECTOR_SHUFFLE()
10514 if (isREVMask(ShuffleMask, VT, 32)) in LowerVECTOR_SHUFFLE()
10516 if (isREVMask(ShuffleMask, VT, 16)) in LowerVECTOR_SHUFFLE()
10519 if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) || in LowerVECTOR_SHUFFLE()
10520 (VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) && in LowerVECTOR_SHUFFLE()
10522 SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1); in LowerVECTOR_SHUFFLE()
10523 return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev, in LowerVECTOR_SHUFFLE()
10529 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) { in LowerVECTOR_SHUFFLE()
10535 } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
10542 if (isZIPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10546 if (isUZPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10550 if (isTRNMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10555 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10559 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10563 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
10582 SrcLane -= VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
10586 EVT ScalarVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE()
10592 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
10602 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
10625 EVT VT = Op.getValueType(); in LowerSPLAT_VECTOR() local
10627 if (useSVEForFixedLengthVectorVT(VT)) in LowerSPLAT_VECTOR()
10630 assert(VT.isScalableVector() && VT.getVectorElementType() == MVT::i1 && in LowerSPLAT_VECTOR()
10646 if (VT == MVT::nxv1i1) in LowerSPLAT_VECTOR()
10651 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, ID, Zero, SplatVal); in LowerSPLAT_VECTOR()
10658 EVT VT = Op.getValueType(); in LowerDUPQLane() local
10659 if (!isTypeLegal(VT) || !VT.isScalableVector()) in LowerDUPQLane()
10663 if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock) in LowerDUPQLane()
10673 return DAG.getNode(AArch64ISD::DUPLANE128, DL, VT, Op.getOperand(1), CI); in LowerDUPQLane()
10696 return DAG.getNode(ISD::BITCAST, DL, VT, TBL); in LowerDUPQLane()
10702 EVT VT = BVN->getValueType(0); in resolveBuildVector() local
10707 unsigned NumSplats = VT.getSizeInBits() / SplatBitSize; in resolveBuildVector()
10712 CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
10713 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
10727 EVT VT = Op.getValueType(); in tryAdvSIMDModImm64() local
10728 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64; in tryAdvSIMDModImm64()
10736 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm64()
10749 EVT VT = Op.getValueType(); in tryAdvSIMDModImm32() local
10750 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in tryAdvSIMDModImm32()
10784 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm32()
10797 EVT VT = Op.getValueType(); in tryAdvSIMDModImm16() local
10798 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in tryAdvSIMDModImm16()
10824 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm16()
10836 EVT VT = Op.getValueType(); in tryAdvSIMDModImm321s() local
10837 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in tryAdvSIMDModImm321s()
10855 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm321s()
10867 EVT VT = Op.getValueType(); in tryAdvSIMDModImm8() local
10868 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; in tryAdvSIMDModImm8()
10876 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm8()
10888 EVT VT = Op.getValueType(); in tryAdvSIMDModImmFP() local
10889 bool isWide = (VT.getSizeInBits() == 128); in tryAdvSIMDModImmFP()
10907 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImmFP()
10925 EVT VT = Bvec->getValueType(0); in isAllConstantBuildVector() local
10926 unsigned NumElts = VT.getVectorNumElements(); in isAllConstantBuildVector()
10941 EVT VT = N->getValueType(0); in tryLowerToSLI() local
10943 if (!VT.isVector()) in tryLowerToSLI()
10997 unsigned ElemSizeInBits = VT.getScalarSizeInBits(); in tryLowerToSLI()
11011 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1)); in tryLowerToSLI()
11031 EVT VT = Op.getValueType(); in LowerVectorOR() local
11044 APInt DefBits(VT.getSizeInBits(), 0); in LowerVectorOR()
11045 APInt UndefBits(VT.getSizeInBits(), 0); in LowerVectorOR()
11072 EVT VT = Op.getValueType(); in NormalizeBuildVector() local
11073 EVT EltTy= VT.getVectorElementType(); in NormalizeBuildVector()
11096 return DAG.getBuildVector(VT, dl, Ops); in NormalizeBuildVector()
11100 EVT VT = Op.getValueType(); in ConstantBuildVector() local
11102 APInt DefBits(VT.getSizeInBits(), 0); in ConstantBuildVector()
11103 APInt UndefBits(VT.getSizeInBits(), 0); in ConstantBuildVector()
11142 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR() local
11144 if (useSVEForFixedLengthVectorVT(VT)) { in LowerBUILD_VECTOR()
11147 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerBUILD_VECTOR()
11160 if (VT.isInteger()) { in LowerBUILD_VECTOR()
11168 unsigned BitSize = VT.getVectorElementType().getSizeInBits(); in LowerBUILD_VECTOR()
11192 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
11235 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
11244 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
11265 if (VT.getVectorElementType() != in LowerBUILD_VECTOR()
11293 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11296 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11300 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
11303 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
11313 Value.getValueType() != VT) { in LowerBUILD_VECTOR()
11316 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value); in LowerBUILD_VECTOR()
11330 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType()); in LowerBUILD_VECTOR()
11331 return DAG.getNode(Opcode, dl, VT, Value, Lane); in LowerBUILD_VECTOR()
11334 if (VT.getVectorElementType().isFloatingPoint()) { in LowerBUILD_VECTOR()
11336 EVT EltTy = VT.getVectorElementType(); in LowerBUILD_VECTOR()
11351 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
11370 SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue), in LowerBUILD_VECTOR()
11374 Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue); in LowerBUILD_VECTOR()
11385 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
11413 SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG); in LowerBUILD_VECTOR()
11418 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector, in LowerBUILD_VECTOR()
11435 SDValue Vec = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
11451 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0); in LowerBUILD_VECTOR()
11461 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
11516 EVT VT = Op.getOperand(0).getValueType(); in LowerINSERT_VECTOR_ELT() local
11518 if (VT.getScalarType() == MVT::i1) { in LowerINSERT_VECTOR_ELT()
11519 EVT VectorVT = getPromotedVTForPredicate(VT); in LowerINSERT_VECTOR_ELT()
11531 return DAG.getAnyExtOrTrunc(ExtendedVector, DL, VT); in LowerINSERT_VECTOR_ELT()
11535 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerINSERT_VECTOR_ELT()
11539 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerINSERT_VECTOR_ELT()
11540 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerINSERT_VECTOR_ELT()
11541 VT == MVT::v8f16 || VT == MVT::v8bf16) in LowerINSERT_VECTOR_ELT()
11544 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerINSERT_VECTOR_ELT()
11545 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 && in LowerINSERT_VECTOR_ELT()
11546 VT != MVT::v4bf16) in LowerINSERT_VECTOR_ELT()
11565 EVT VT = Op.getOperand(0).getValueType(); in LowerEXTRACT_VECTOR_ELT() local
11567 if (VT.getScalarType() == MVT::i1) { in LowerEXTRACT_VECTOR_ELT()
11570 EVT VectorVT = getPromotedVTForPredicate(VT); in LowerEXTRACT_VECTOR_ELT()
11580 if (useSVEForFixedLengthVectorVT(VT)) in LowerEXTRACT_VECTOR_ELT()
11585 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerEXTRACT_VECTOR_ELT()
11589 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerEXTRACT_VECTOR_ELT()
11590 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerEXTRACT_VECTOR_ELT()
11591 VT == MVT::v8f16 || VT == MVT::v8bf16) in LowerEXTRACT_VECTOR_ELT()
11594 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerEXTRACT_VECTOR_ELT()
11595 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 && in LowerEXTRACT_VECTOR_ELT()
11596 VT != MVT::v4bf16) in LowerEXTRACT_VECTOR_ELT()
11671 EVT VT = Op.getValueType(); in LowerINSERT_SUBVECTOR() local
11674 if (!isTypeLegal(VT)) in LowerINSERT_SUBVECTOR()
11678 if (VT.getVectorElementType() == MVT::i1) { in LowerINSERT_SUBVECTOR()
11679 unsigned NumElts = VT.getVectorMinNumElements(); in LowerINSERT_SUBVECTOR()
11680 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in LowerINSERT_SUBVECTOR()
11690 return DAG.getNode(AArch64ISD::UZP1, DL, VT, NewLo, Hi); in LowerINSERT_SUBVECTOR()
11695 return DAG.getNode(AArch64ISD::UZP1, DL, VT, Lo, NewHi); in LowerINSERT_SUBVECTOR()
11700 if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2)) in LowerINSERT_SUBVECTOR()
11706 EVT NarrowVT = getPackedSVEVectorVT(VT.getVectorElementCount()); in LowerINSERT_SUBVECTOR()
11710 if (VT.isFloatingPoint()) { in LowerINSERT_SUBVECTOR()
11732 return getSVESafeBitCast(VT, Narrow, DAG); in LowerINSERT_SUBVECTOR()
11735 if (Idx == 0 && isPackedVectorType(VT, DAG)) { in LowerINSERT_SUBVECTOR()
11742 auto PredTy = VT.changeVectorElementType(MVT::i1); in LowerINSERT_SUBVECTOR()
11744 SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1); in LowerINSERT_SUBVECTOR()
11745 return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0); in LowerINSERT_SUBVECTOR()
11783 EVT VT = Op.getValueType(); in LowerDIV() local
11786 if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true)) in LowerDIV()
11789 assert(VT.isScalableVector() && "Expected a scalable vector."); in LowerDIV()
11797 SDValue Pg = getPredicateForScalableVector(DAG, dl, VT); in LowerDIV()
11799 DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, VT, Pg, Op->getOperand(0), in LowerDIV()
11802 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerDIV()
11807 if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64) in LowerDIV()
11813 if (VT == MVT::nxv16i8) in LowerDIV()
11815 else if (VT == MVT::nxv8i16) in LowerDIV()
11828 return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLo, ResultHi); in LowerDIV()
11831 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { in isShuffleMaskLegal()
11833 if (useSVEForFixedLengthVectorVT(VT)) in isShuffleMaskLegal()
11836 if (VT.getVectorNumElements() == 4 && in isShuffleMaskLegal()
11837 (VT.is128BitVector() || VT.is64BitVector())) { in isShuffleMaskLegal()
11847 return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) || in isShuffleMaskLegal()
11848 isREVMask(M, VT, 32) || isREVMask(M, VT, 16) || in isShuffleMaskLegal()
11849 isEXTMask(M, VT, DummyBool, DummyUnsigned) || in isShuffleMaskLegal()
11851 isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
11852 isZIPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
11853 isTRN_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
11854 isUZP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
11855 isZIP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
11856 isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) || in isShuffleMaskLegal()
11857 isConcatMask(M, VT, VT.getSizeInBits() == 128)); in isShuffleMaskLegal()
11861 EVT VT) const { in isVectorClearMaskLegal()
11863 return isShuffleMaskLegal(M, VT); in isVectorClearMaskLegal()
11889 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { in isVShiftLImm() argument
11890 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftLImm()
11891 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftLImm()
11900 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) { in isVShiftRImm() argument
11901 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftRImm()
11902 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftRImm()
11910 EVT VT = Op.getValueType(); in LowerTRUNCATE() local
11912 if (VT.getScalarType() == MVT::i1) { in LowerTRUNCATE()
11919 return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE); in LowerTRUNCATE()
11922 if (!VT.isVector() || VT.isScalableVector()) in LowerTRUNCATE()
11933 EVT VT = Op.getValueType(); in LowerVectorSRA_SRL_SHL() local
11939 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVectorSRA_SRL_SHL()
11943 if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) in LowerVectorSRA_SRL_SHL()
11946 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) in LowerVectorSRA_SRL_SHL()
11947 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
11949 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
11955 if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) { in LowerVectorSRA_SRL_SHL()
11962 if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) { in LowerVectorSRA_SRL_SHL()
11965 return DAG.getNode(Opc, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
11975 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerVectorSRA_SRL_SHL()
11978 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
11988 AArch64CC::CondCode CC, bool NoNans, EVT VT, in EmitVectorComparison() argument
11991 assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && in EmitVectorComparison()
11995 APInt CnstBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
11996 APInt UndefBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
12007 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
12009 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12010 return DAG.getNOT(dl, Fcmeq, VT); in EmitVectorComparison()
12014 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
12015 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12018 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS); in EmitVectorComparison()
12019 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
12022 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS); in EmitVectorComparison()
12023 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
12031 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS); in EmitVectorComparison()
12032 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
12040 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); in EmitVectorComparison()
12041 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
12051 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
12053 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12054 return DAG.getNOT(dl, Cmeq, VT); in EmitVectorComparison()
12058 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
12059 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12062 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS); in EmitVectorComparison()
12063 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
12066 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); in EmitVectorComparison()
12067 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
12070 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); in EmitVectorComparison()
12071 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
12073 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS); in EmitVectorComparison()
12075 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS); in EmitVectorComparison()
12078 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); in EmitVectorComparison()
12079 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
12081 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); in EmitVectorComparison()
12083 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS); in EmitVectorComparison()
12249 MVT VT = Op.getSimpleValueType(); in LowerATOMIC_LOAD_SUB() local
12252 RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS); in LowerATOMIC_LOAD_SUB()
12266 MVT VT = Op.getSimpleValueType(); in LowerATOMIC_LOAD_AND() local
12269 RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS); in LowerATOMIC_LOAD_AND()
12315 EVT VT = Node->getValueType(0); in LowerDYNAMIC_STACKALLOC() local
12323 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
12324 DAG.getConstant(-(uint64_t)Align->value(), dl, VT)); in LowerDYNAMIC_STACKALLOC()
12338 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
12339 DAG.getConstant(-(uint64_t)Align->value(), dl, VT)); in LowerDYNAMIC_STACKALLOC()
12351 EVT VT = Op.getValueType(); in LowerVSCALE() local
12352 assert(VT != MVT::i64 && "Expected illegal VSCALE node"); in LowerVSCALE()
12357 VT); in LowerVSCALE()
12367 const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType()); in setInfoSVEStN() local
12368 ElementCount EC = VT.getVectorElementCount(); in setInfoSVEStN()
12372 assert(VT == TLI.getMemValueType(DL, CI.getArgOperand(I)->getType()) && in setInfoSVEStN()
12376 Info.memVT = EVT::getVectorVT(CI.getType()->getContext(), VT.getScalarType(), in setInfoSVEStN()
13380 EVT VT, SelectionDAG &DAG, in LowerSVEStructLoad() argument
13382 assert(VT.isScalableVector() && "Can only lower scalable vectors"); in LowerSVEStructLoad()
13394 assert(VT.getVectorElementCount().getKnownMinValue() % N == 0 && in LowerSVEStructLoad()
13398 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in LowerSVEStructLoad()
13399 VT.getVectorElementCount().divideCoefficientBy(N)); in LowerSVEStructLoad()
13410 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps); in LowerSVEStructLoad()
13422 auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) { in getOptimalMemOpType() argument
13426 return allowsMisalignedMemoryAccesses(VT, 0, Align(1), in getOptimalMemOpType()
13452 auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) { in getOptimalMemOpLLT() argument
13456 return allowsMisalignedMemoryAccesses(VT, 0, Align(1), in getOptimalMemOpLLT()
13495 const EVT VT = AddNode.getValueType(); in isMulAddWithConstProfitable() local
13496 if (VT.isVector() || VT.getScalarSizeInBits() > 64) in isMulAddWithConstProfitable()
13508 AArch64_IMM::expandMOVImm(C1C2.getZExtValue(), VT.getSizeInBits(), Insn); in isMulAddWithConstProfitable()
13603 const MachineFunction &MF, EVT VT) const { in isFMAFasterThanFMulAndFAdd()
13604 VT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
13606 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
13609 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
13634 EVT VT, CodeGenOpt::Level OptLevel) const { in generateFMAsInMachineCombiner() argument
13635 return (OptLevel >= CodeGenOpt::Aggressive) && !VT.isScalableVector() && in generateFMAsInMachineCombiner()
13636 !useSVEForFixedLengthVectorVT(VT); in generateFMAsInMachineCombiner()
13658 EVT VT = N->getValueType(0); in isDesirableToCommuteWithShift() local
13662 if (ShiftLHS.getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
13709 EVT VT = N->getValueType(0); in shouldFoldConstantShiftPairToMask() local
13710 if (N->getOpcode() == ISD::SRL && (VT == MVT::i32 || VT == MVT::i64)) { in shouldFoldConstantShiftPairToMask()
13756 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp() local
13757 if (!Subtarget->hasNEON() || !VT.isVector()) in foldVectorXorShiftIntoCmp()
13774 return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0)); in foldVectorXorShiftIntoCmp()
13925 EVT VT = A.getValueType(); in performUADDVCombine() local
13941 VT.getVectorNumElements() * 2) in performUADDVCombine()
13944 Ext1.getConstantOperandVal(1) != VT.getVectorNumElements()) && in performUADDVCombine()
13946 Ext0.getConstantOperandVal(1) != VT.getVectorNumElements())) in performUADDVCombine()
13950 return DAG.getNode(Opcode, SDLoc(A), VT, Ext0.getOperand(0)); in performUADDVCombine()
13987 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
13991 if (VT.isScalableVector() || Subtarget->useSVEForFixedLengthVectors()) in BuildSDIVPow2()
13995 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSDIVPow2()
14002 SDValue Zero = DAG.getConstant(0, DL, VT); in BuildSDIVPow2()
14003 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT); in BuildSDIVPow2()
14008 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
14009 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp); in BuildSDIVPow2()
14017 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
14025 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
14036 EVT VT = N->getValueType(0); in BuildSREMPow2() local
14040 if (VT.isScalableVector() || Subtarget->useSVEForFixedLengthVectors()) in BuildSREMPow2()
14044 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSREMPow2()
14054 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT); in BuildSREMPow2()
14055 SDValue Zero = DAG.getConstant(0, DL, VT); in BuildSREMPow2()
14059 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14060 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, And, And, CCVal, Cmp); in BuildSREMPow2()
14066 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in BuildSREMPow2()
14069 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14070 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne); in BuildSREMPow2()
14071 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, AndPos, AndNeg, CCVal, in BuildSREMPow2()
14145 EVT VT = BV.getValueType(); in performBuildShuffleExtendCombine() local
14169 PreExtendType.getScalarSizeInBits() != VT.getScalarSizeInBits() / 2) in performBuildShuffleExtendCombine()
14186 EVT PreExtendVT = VT.changeVectorElementType(PreExtendType); in performBuildShuffleExtendCombine()
14196 EVT PreExtendVT = VT.changeVectorElementType(PreExtendType.getScalarType()); in performBuildShuffleExtendCombine()
14203 return DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT, NBV); in performBuildShuffleExtendCombine()
14210 EVT VT = Mul->getValueType(0); in performMulVectorExtendCombine() local
14211 if (VT != MVT::v8i16 && VT != MVT::v4i32 && VT != MVT::v2i64) in performMulVectorExtendCombine()
14222 return DAG.getNode(Mul->getOpcode(), DL, VT, Op0 ? Op0 : Mul->getOperand(0), in performMulVectorExtendCombine()
14240 EVT VT = N->getValueType(0); in performMulCombine() local
14260 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMulCombine()
14261 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal); in performMulCombine()
14265 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMulCombine()
14266 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal); in performMulCombine()
14351 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0, in performMulCombine()
14356 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1); in performMulCombine()
14361 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); in performMulCombine()
14364 return DAG.getNode(ISD::SHL, DL, VT, Res, in performMulCombine()
14382 EVT VT = N->getValueType(0); in performVectorCompareAndMaskUnaryOpCombine() local
14383 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
14385 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits()) in performVectorCompareAndMaskUnaryOpCombine()
14403 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in performVectorCompareAndMaskUnaryOpCombine()
14408 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
14422 EVT VT = N->getValueType(0); in performIntToFpCombine() local
14423 if (VT != MVT::f32 && VT != MVT::f64) in performIntToFpCombine()
14427 if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits()) in performIntToFpCombine()
14438 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in performIntToFpCombine()
14448 return DAG.getNode(Opcode, SDLoc(N), VT, Load); in performIntToFpCombine()
14622 EVT VT = N->getValueType(0); in tryCombineToEXTR() local
14626 if (VT != MVT::i32 && VT != MVT::i64) in tryCombineToEXTR()
14646 if (ShiftLHS + ShiftRHS != VT.getSizeInBits()) in tryCombineToEXTR()
14654 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS, in tryCombineToEXTR()
14660 EVT VT = N->getValueType(0); in tryCombineToBSL() local
14664 if (!VT.isVector()) in tryCombineToBSL()
14669 if (!VT.is64BitVector() && !VT.is128BitVector()) in tryCombineToBSL()
14713 return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling); in tryCombineToBSL()
14720 unsigned Bits = VT.getScalarSizeInBits(); in tryCombineToBSL()
14730 for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) { in tryCombineToBSL()
14741 return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0), in tryCombineToBSL()
14759 EVT VT = N->getValueType(0); in performANDORCSELCombine() local
14810 return DAG.getNode(AArch64ISD::CSEL, DL, VT, CSel0.getOperand(0), in performANDORCSELCombine()
14818 EVT VT = N->getValueType(0); in performORCombine() local
14823 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in performORCombine()
14959 EVT VT = N->getValueType(0); in performANDCombine() local
14964 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in performANDCombine()
14967 if (VT.isScalableVector()) in performANDCombine()
14972 if (!VT.is64BitVector() && !VT.is128BitVector()) in performANDCombine()
14983 APInt DefBits(VT.getSizeInBits(), 0); in performANDCombine()
14984 APInt UndefBits(VT.getSizeInBits(), 0); in performANDCombine()
15006 static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16) { in hasPairwiseAdd() argument
15010 return (FullFP16 && VT == MVT::f16) || VT == MVT::f32 || VT == MVT::f64; in hasPairwiseAdd()
15012 return VT == MVT::i64; in hasPairwiseAdd()
15018 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
15051 EVT VT = N0.getValueType(); in performFirstTrueTestVectorCombine() local
15053 if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1 || in performFirstTrueTestVectorCombine()
15064 SDValue Pg = getPTrue(DAG, SDLoc(N), VT, AArch64SVEPredPattern::all); in performFirstTrueTestVectorCombine()
15118 EVT VT = N->getValueType(0); in performExtractVectorEltCombine() local
15124 return DAG.getZExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in performExtractVectorEltCombine()
15136 hasPairwiseAdd(N0->getOpcode(), VT, FullFP16) && in performExtractVectorEltCombine()
15153 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15155 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15158 return DAG.getNode(N0->getOpcode(), DL, VT, Extract1, Extract2); in performExtractVectorEltCombine()
15165 {VT, MVT::Other}, in performExtractVectorEltCombine()
15180 EVT VT = N->getValueType(0); in performConcatVectorsCombine() local
15184 if (VT.isScalableVector()) in performConcatVectorsCombine()
15206 N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) { in performConcatVectorsCombine()
15211 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
15292 N00Source.getValueType() == VT && N01Source.getValueType() == VT) { in performConcatVectorsCombine()
15302 return DAG.getNode(N0Opc, dl, VT, N00Source, N01Source); in performConcatVectorsCombine()
15310 if (N->getNumOperands() == 2 && N0 == N1 && VT.getVectorNumElements() == 2) { in performConcatVectorsCombine()
15311 assert(VT.getScalarSizeInBits() == 64); in performConcatVectorsCombine()
15312 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), in performConcatVectorsCombine()
15338 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
15350 EVT VT = N->getValueType(0); in performExtractSubvectorCombine() local
15351 if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1) in performExtractSubvectorCombine()
15362 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V.getOperand(0)); in performExtractSubvectorCombine()
15473 MVT VT = N.getSimpleValueType(); in tryExtendDUPToExtractHigh() local
15498 if (!VT.is64BitVector()) in tryExtendDUPToExtractHigh()
15502 unsigned NumElems = VT.getVectorNumElements(); in tryExtendDUPToExtractHigh()
15504 MVT ElementTy = VT.getVectorElementType(); in tryExtendDUPToExtractHigh()
15509 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N, in tryExtendDUPToExtractHigh()
15654 EVT VT = Op->getValueType(0); in performSetccAddFolding() local
15655 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
15656 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp); in performSetccAddFolding()
15661 EVT VT = N->getValueType(0); in performAddUADDVCombine() local
15663 if (N->getOpcode() != ISD::ADD || !VT.isScalarInteger()) in performAddUADDVCombine()
15669 RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || LHS.getValueType() != VT) in performAddUADDVCombine()
15683 OpVT1.getVectorElementType() != VT) in performAddUADDVCombine()
15691 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in performAddUADDVCombine()
15700 EVT VT = N->getValueType(0); in performAddCSelIntoCSinc() local
15701 if (!VT.isScalarInteger() || N->getOpcode() != ISD::ADD) in performAddCSelIntoCSinc()
15748 CTVal = cast<ConstantSDNode>(DAG.getConstant(C, DL, VT)); in performAddCSelIntoCSinc()
15749 CFVal = cast<ConstantSDNode>(DAG.getAllOnesConstant(DL, VT)); in performAddCSelIntoCSinc()
15764 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0)); in performAddCSelIntoCSinc()
15768 return DAG.getNode(AArch64ISD::CSINC, DL, VT, NewNode, RHS, CCVal, Cmp); in performAddCSelIntoCSinc()
15773 EVT VT = N->getValueType(0); in performAddDotCombine() local
15790 return DAG.getNode(Dot.getOpcode(), SDLoc(N), VT, A, Dot.getOperand(1), in performAddDotCombine()
15800 EVT VT = Op.getValueType(); in getNegatedInteger() local
15801 SDValue Zero = DAG.getConstant(0, DL, VT); in getNegatedInteger()
15802 return DAG.getNode(ISD::SUB, DL, VT, Zero, Op); in getNegatedInteger()
15832 EVT VT = CSel.getValueType(); in performNegCSelCombine() local
15833 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0N, N1N, CSel.getOperand(2), in performNegCSelCombine()
15854 MVT VT = N->getSimpleValueType(0); in performAddSubLongCombine() local
15855 if (!VT.is128BitVector()) { in performAddSubLongCombine()
15878 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); in performAddSubLongCombine()
15884 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS); in performAddSubLongCombine()
15887 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS); in performAddSubLongCombine()
15947 EVT VT = N->getValueType(0); in foldADCToCINC() local
15952 return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond); in foldADCToCINC()
15960 EVT VT = N->getValueType(0); in performVectorAddSubExtCombine() local
15961 if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 || in performVectorAddSubExtCombine()
15975 EVT S2 = VT.getScalarType(); in performVectorAddSubExtCombine()
15981 VT.getVectorElementCount()); in performVectorAddSubExtCombine()
15985 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewOp); in performVectorAddSubExtCombine()
16217 EVT VT = N->getValueType(0); in LowerSVEIntrinsicEXT() local
16219 assert(VT.isScalableVector() && "Expected a scalable vector."); in LowerSVEIntrinsicEXT()
16222 if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock) in LowerSVEIntrinsicEXT()
16225 unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8; in LowerSVEIntrinsicEXT()
16226 unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8; in LowerSVEIntrinsicEXT()
16237 return DAG.getNode(ISD::BITCAST, dl, VT, EXT); in LowerSVEIntrinsicEXT()
16250 EVT VT = N->getValueType(0); in tryConvertSVEWideCompare() local
16297 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred, in tryConvertSVEWideCompare()
16304 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, in getPTest() argument
16316 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in getPTest()
16337 return DAG.getZExtOrTrunc(Res, DL, VT); in getPTest()
16480 EVT VT = N->getValueType(0); in performIntrinsicCombine() local
16481 if (VT.isFixedLengthVector()) { in performIntrinsicCombine()
16492 ElementCount::getScalable(VT.getVectorNumElements())); in performIntrinsicCombine()
16500 VT.getVectorElementCount()); in performIntrinsicCombine()
16507 Res = DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in performIntrinsicCombine()
16893 EVT VT = N->getValueType(0); in performLD1Combine() local
16895 if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock) in performLD1Combine()
16898 EVT ContainerVT = VT; in performLD1Combine()
16906 DAG.getValueType(VT) }; in performLD1Combine()
16911 if (ContainerVT.isInteger() && (VT != ContainerVT)) in performLD1Combine()
16912 Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0)); in performLD1Combine()
16919 EVT VT = N->getValueType(0); in performLDNT1Combine() local
16922 EVT LoadVT = VT; in performLDNT1Combine()
16923 if (VT.isFloatingPoint()) in performLDNT1Combine()
16924 LoadVT = VT.changeTypeToInteger(); in performLDNT1Combine()
16934 if (VT.isFloatingPoint()) { in performLDNT1Combine()
16935 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) }; in performLDNT1Combine()
16948 EVT VT = N->getValueType(0); in performLD1ReplicateCombine() local
16950 EVT LoadVT = VT; in performLD1ReplicateCombine()
16951 if (VT.isFloatingPoint()) in performLD1ReplicateCombine()
16952 LoadVT = VT.changeTypeToInteger(); in performLD1ReplicateCombine()
16958 if (VT.isFloatingPoint()) in performLD1ReplicateCombine()
16959 Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0)); in performLD1ReplicateCombine()
17024 EVT VT = StVal.getValueType(); in replaceZeroVectorStore() local
17027 if (VT.isScalableVector()) in replaceZeroVectorStore()
17032 int NumVecElts = VT.getVectorNumElements(); in replaceZeroVectorStore()
17034 VT.getVectorElementType().getSizeInBits() == 64) || in replaceZeroVectorStore()
17036 VT.getVectorElementType().getSizeInBits() == 32))) in replaceZeroVectorStore()
17072 if (VT.getVectorElementType().getSizeInBits() == 32) { in replaceZeroVectorStore()
17091 EVT VT = StVal.getValueType(); in replaceSplatVectorStore() local
17095 if (VT.isFloatingPoint()) in replaceSplatVectorStore()
17099 unsigned NumVecElts = VT.getVectorNumElements(); in replaceSplatVectorStore()
17151 EVT VT = StVal.getValueType(); in splitStores() local
17153 if (!VT.isFixedLengthVector()) in splitStores()
17175 if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64) in splitStores()
17183 if (VT.getSizeInBits() != 128 || S->getAlign() >= Align(16) || in splitStores()
17196 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in splitStores()
17248 EVT VT = N->getValueType(0); in performUnpackCombine() local
17253 NumElts * VT.getVectorElementType().getSizeInBits() <= MinSVESize) { in performUnpackCombine()
17255 getPTrue(DAG, DL, VT.changeVectorElementType(MVT::i1), PgPattern); in performUnpackCombine()
17256 SDValue PassThru = DAG.getConstant(0, DL, VT); in performUnpackCombine()
17258 VT, DL, MLD->getChain(), MLD->getBasePtr(), MLD->getOffset(), Mask, in performUnpackCombine()
17412 auto VT = CC->getValueType(0).getHalfNumVectorElementsVT(*DAG.getContext()); in performSunpkloCombine() local
17413 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC, in performSunpkloCombine()
17430 EVT VT = N->getValueType(0); in performPostLD1Combine() local
17432 if (!VT.is128BitVector() && !VT.is64BitVector()) in performPostLD1Combine()
17446 if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements()) in performPostLD1Combine()
17453 if (MemVT != VT.getVectorElementType()) in performPostLD1Combine()
17480 unsigned NumBytes = VT.getScalarSizeInBits() / 8; in performPostLD1Combine()
17507 EVT Tys[3] = { VT, MVT::i64, MVT::Other }; in performPostLD1Combine()
18361 EVT VT = N->getValueType(0); in performSETCCCombine() local
18381 return DAG.getZExtOrTrunc(CSEL, DL, VT); in performSETCCCombine()
18394 return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2)); in performSETCCCombine()
18400 if (DCI.isBeforeLegalize() && VT.isScalarInteger() && in performSETCCCombine()
18409 return DAG.getSetCC(DL, VT, LHS, RHS, Cond); in performSETCCCombine()
18424 EVT VT = N->getValueType(0); in performFlagSettingCombine() local
18428 SDValue Res = DCI.DAG.getNode(GenericOpcode, DL, VT, N->ops()); in performFlagSettingCombine()
18435 GenericOpcode, DCI.DAG.getVTList(VT), {LHS, RHS})) in performFlagSettingCombine()
18688 EVT VT = CmpLHS.getValueType(); in performVSelectCombine() local
18694 VT.isSimple() && in performVSelectCombine()
18698 VT.getSimpleVT().SimpleTy) && in performVSelectCombine()
18702 unsigned NumElts = VT.getVectorNumElements(); in performVSelectCombine()
18704 NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N), in performVSelectCombine()
18705 VT.getScalarType())); in performVSelectCombine()
18706 SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops); in performVSelectCombine()
18708 auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val); in performVSelectCombine()
18709 auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1)); in performVSelectCombine()
18804 EVT VT = N->getValueType(0); in performDUPCombine() local
18807 if (VT.is64BitVector() && DCI.isAfterLegalizeDAG()) { in performDUPCombine()
18808 EVT LVT = VT.getDoubleNumVectorElementsVT(*DCI.DAG.getContext()); in performDUPCombine()
18812 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0), in performDUPCombine()
19156 auto VT = cast<VTSDNode>(N->getOperand(1))->getVT(); in performSignExtendInRegCombine() local
19157 EVT EltTy = VT.getVectorElementType(); in performSignExtendInRegCombine()
19163 EVT ExtVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext()); in performSignExtendInRegCombine()
19404 EVT VT = N->getValueType(0); in performFPExtendCombine() local
19415 VT.isFixedLengthVector() && in performFPExtendCombine()
19416 VT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits()) { in performFPExtendCombine()
19418 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in performFPExtendCombine()
19435 EVT VT = N->getValueType(0); in performBSPExpandForSVE() local
19438 if (!VT.isScalableVector() || Subtarget->hasSVE2() || Subtarget->hasSME()) in performBSPExpandForSVE()
19442 if (VT.isFixedLengthVector() && !fixedSVEVectorVT) in performBSPExpandForSVE()
19451 SDValue InvMask = DAG.getNOT(DL, Mask, VT); in performBSPExpandForSVE()
19452 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1); in performBSPExpandForSVE()
19453 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2); in performBSPExpandForSVE()
19454 return DAG.getNode(ISD::OR, DL, VT, Sel, SelInv); in performBSPExpandForSVE()
19458 EVT VT = N->getValueType(0); in performDupLane128Combine() local
19489 return DAG.getNode(ISD::BITCAST, DL, VT, NewDuplane128); in performDupLane128Combine()
19826 EVT VT = Opnds[0].getValueType(); in PerformDAGCombine() local
19827 EVT EltVT = VT.getVectorElementType(); in PerformDAGCombine()
19829 VT.getVectorElementCount() * in PerformDAGCombine()
19949 EVT VT; in getPreIndexedAddressParts() local
19952 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19955 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19970 EVT VT; in getPostIndexedAddressParts() local
19973 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
19976 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
19996 EVT VT = N->getValueType(0); in ReplaceBITCASTResults() local
19999 if (VT.isScalableVector() && !isTypeLegal(VT) && isTypeLegal(SrcVT)) { in ReplaceBITCASTResults()
20000 assert(!VT.isFloatingPoint() && SrcVT.isFloatingPoint() && in ReplaceBITCASTResults()
20008 if (VT.getVectorElementCount() != SrcVT.getVectorElementCount()) in ReplaceBITCASTResults()
20011 SDValue CastResult = getSVESafeBitCast(getSVEContainerType(VT), Op, DAG); in ReplaceBITCASTResults()
20012 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, CastResult)); in ReplaceBITCASTResults()
20016 if (VT != MVT::i16 || (SrcVT != MVT::f16 && SrcVT != MVT::bf16)) in ReplaceBITCASTResults()
20031 EVT VT = N->getValueType(0); in ReplaceAddWithADDP() local
20032 if (!VT.is256BitVector() || in ReplaceAddWithADDP()
20033 (VT.getScalarType().isFloatingPoint() && in ReplaceAddWithADDP()
20035 (VT.getScalarType() == MVT::f16 && !Subtarget->hasFullFP16())) in ReplaceAddWithADDP()
20064 for (unsigned I = 0, E = VT.getVectorNumElements() / 2; I < E; I++) { in ReplaceAddWithADDP()
20069 DAG.getVectorShuffle(VT, DL, in ReplaceAddWithADDP()
20070 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Addp, in ReplaceAddWithADDP()
20072 DAG.getUNDEF(VT), NMask)); in ReplaceAddWithADDP()
20108 EVT VT = N->getValueType(0); in ReplaceExtractSubVectorResults() local
20112 ElementCount ResEC = VT.getVectorElementCount(); in ReplaceExtractSubVectorResults()
20126 EVT ExtendedHalfVT = VT.widenIntegerVectorElementType(*DAG.getContext()); in ReplaceExtractSubVectorResults()
20129 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half)); in ReplaceExtractSubVectorResults()
20323 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
20324 assert((VT == MVT::i8 || VT == MVT::i16) && in ReplaceNodeResults()
20337 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20345 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20352 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20359 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20380 AArch64TargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
20383 if (VT == MVT::v1i8 || VT == MVT::v1i16 || VT == MVT::v1i32 || in getPreferredVectorAction()
20384 VT == MVT::v1f32) in getPreferredVectorAction()
20387 return TargetLoweringBase::getPreferredVectorAction(VT); in getPreferredVectorAction()
20768 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { in isIntDivCheap() argument
20777 return OptSize && !VT.isVector(); in isIntDivCheap()
20780 bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { in preferIncOfAddToSubOfNot()
20782 return VT.isScalarInteger(); in preferIncOfAddToSubOfNot()
20786 EVT VT) const { in shouldConvertFpToSat()
20791 return TargetLowering::shouldConvertFpToSat(Op, FPVT, VT); in shouldConvertFpToSat()
20794 bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const { in enableAggressiveFMAFusion()
20795 return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint(); in enableAggressiveFMAFusion()
20874 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT) { in getContainerForFixedLengthVector() argument
20875 assert(VT.isFixedLengthVector() && in getContainerForFixedLengthVector()
20876 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in getContainerForFixedLengthVector()
20878 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) { in getContainerForFixedLengthVector()
20900 EVT VT) { in getPredicateForFixedLengthVector() argument
20901 assert(VT.isFixedLengthVector() && in getPredicateForFixedLengthVector()
20902 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in getPredicateForFixedLengthVector()
20906 getSVEPredPatternFromNumElements(VT.getVectorNumElements()); in getPredicateForFixedLengthVector()
20916 MaxSVESize == VT.getSizeInBits()) in getPredicateForFixedLengthVector()
20920 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) { in getPredicateForFixedLengthVector()
20944 EVT VT) { in getPredicateForScalableVector() argument
20945 assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) && in getPredicateForScalableVector()
20947 auto PredTy = VT.changeVectorElementType(MVT::i1); in getPredicateForScalableVector()
20951 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) { in getPredicateForVector() argument
20952 if (VT.isFixedLengthVector()) in getPredicateForVector()
20953 return getPredicateForFixedLengthVector(DAG, DL, VT); in getPredicateForVector()
20955 return getPredicateForScalableVector(DAG, DL, VT); in getPredicateForVector()
20959 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) { in convertToScalableVector() argument
20960 assert(VT.isScalableVector() && in convertToScalableVector()
20966 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
20970 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) { in convertFromScalableVector() argument
20971 assert(VT.isFixedLengthVector() && in convertFromScalableVector()
20977 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
20986 EVT VT = Op.getValueType(); in LowerFixedLengthVectorLoadToSVE() local
20987 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorLoadToSVE()
20991 auto Pg = getPredicateForFixedLengthVector(DAG, DL, VT); in LowerFixedLengthVectorLoadToSVE()
20993 if (VT.isFloatingPoint() && Load->getExtensionType() == ISD::EXTLOAD) { in LowerFixedLengthVectorLoadToSVE()
21004 if (VT.isFloatingPoint() && Load->getExtensionType() == ISD::EXTLOAD) { in LowerFixedLengthVectorLoadToSVE()
21013 Result = convertFromScalableVector(DAG, VT, Result); in LowerFixedLengthVectorLoadToSVE()
21042 EVT VT = Op.getValueType(); in LowerFixedLengthVectorMLoadToSVE() local
21043 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorMLoadToSVE()
21074 Result = convertFromScalableVector(DAG, VT, Result); in LowerFixedLengthVectorMLoadToSVE()
21085 EVT VT = Store->getValue().getValueType(); in LowerFixedLengthVectorStoreToSVE() local
21086 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorStoreToSVE()
21089 auto Pg = getPredicateForFixedLengthVector(DAG, DL, VT); in LowerFixedLengthVectorStoreToSVE()
21092 if (VT.isFloatingPoint() && Store->isTruncatingStore()) { in LowerFixedLengthVectorStoreToSVE()
21114 EVT VT = Store->getValue().getValueType(); in LowerFixedLengthVectorMStoreToSVE() local
21115 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorMStoreToSVE()
21129 EVT VT = Op.getValueType(); in LowerFixedLengthVectorIntDivideToSVE() local
21130 EVT EltVT = VT.getVectorElementType(); in LowerFixedLengthVectorIntDivideToSVE()
21138 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorIntDivideToSVE()
21142 SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT); in LowerFixedLengthVectorIntDivideToSVE()
21145 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerFixedLengthVectorIntDivideToSVE()
21147 return convertFromScalableVector(DAG, VT, Res); in LowerFixedLengthVectorIntDivideToSVE()
21155 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVectorIntDivideToSVE()
21156 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in LowerFixedLengthVectorIntDivideToSVE()
21161 EVT WidenedVT = VT.widenIntegerVectorElementType(*DAG.getContext()); in LowerFixedLengthVectorIntDivideToSVE()
21167 return DAG.getNode(ISD::TRUNCATE, dl, VT, Div); in LowerFixedLengthVectorIntDivideToSVE()
21198 return convertFromScalableVector(DAG, VT, ScalableResult); in LowerFixedLengthVectorIntDivideToSVE()
21203 EVT VT = Op.getValueType(); in LowerFixedLengthVectorIntExtendToSVE() local
21204 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthVectorIntExtendToSVE()
21220 if (VT.getVectorElementType() == MVT::i16) in LowerFixedLengthVectorIntExtendToSVE()
21225 if (VT.getVectorElementType() == MVT::i32) in LowerFixedLengthVectorIntExtendToSVE()
21230 assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!"); in LowerFixedLengthVectorIntExtendToSVE()
21234 return convertFromScalableVector(DAG, VT, Val); in LowerFixedLengthVectorIntExtendToSVE()
21239 EVT VT = Op.getValueType(); in LowerFixedLengthVectorTruncateToSVE() local
21240 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthVectorTruncateToSVE()
21254 if (VT.getVectorElementType() == MVT::i32) in LowerFixedLengthVectorTruncateToSVE()
21260 if (VT.getVectorElementType() == MVT::i16) in LowerFixedLengthVectorTruncateToSVE()
21266 assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!"); in LowerFixedLengthVectorTruncateToSVE()
21270 return convertFromScalableVector(DAG, VT, Val); in LowerFixedLengthVectorTruncateToSVE()
21275 EVT VT = Op.getValueType(); in LowerFixedLengthExtractVectorElt() local
21283 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1)); in LowerFixedLengthExtractVectorElt()
21288 EVT VT = Op.getValueType(); in LowerFixedLengthInsertVectorElt() local
21289 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthInsertVectorElt()
21299 return convertFromScalableVector(DAG, VT, ScalableRes); in LowerFixedLengthInsertVectorElt()
21308 EVT VT = Op.getValueType(); in LowerToPredicatedOp() local
21310 auto Pg = getPredicateForVector(DAG, DL, VT); in LowerToPredicatedOp()
21312 if (VT.isFixedLengthVector()) { in LowerToPredicatedOp()
21313 assert(isTypeLegal(VT) && "Expected only legal fixed-width types"); in LowerToPredicatedOp()
21314 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerToPredicatedOp()
21340 return convertFromScalableVector(DAG, VT, ScalableRes); in LowerToPredicatedOp()
21343 assert(VT.isScalableVector() && "Only expect to lower scalable vector op!"); in LowerToPredicatedOp()
21354 Operands.push_back(DAG.getUNDEF(VT)); in LowerToPredicatedOp()
21356 return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags()); in LowerToPredicatedOp()
21364 EVT VT = Op.getValueType(); in LowerToScalableOp() local
21365 assert(useSVEForFixedLengthVectorVT(VT) && in LowerToScalableOp()
21367 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerToScalableOp()
21387 return convertFromScalableVector(DAG, VT, ScalableRes); in LowerToScalableOp()
21423 EVT VT = ReduceOp.getValueType(); in LowerPredReductionToSVE() local
21437 return getPTest(DAG, VT, Op, Op, AArch64CC::ANY_ACTIVE); in LowerPredReductionToSVE()
21439 return getPTest(DAG, VT, Pg, Op, AArch64CC::ANY_ACTIVE); in LowerPredReductionToSVE()
21442 return getPTest(DAG, VT, Pg, Op, AArch64CC::NONE_ACTIVE); in LowerPredReductionToSVE()
21454 return DAG.getAnyExtOrTrunc(Cntp, DL, VT); in LowerPredReductionToSVE()
21497 EVT VT = Op.getValueType(); in LowerFixedLengthVectorSelectToSVE() local
21516 return convertFromScalableVector(DAG, VT, ScalableRes); in LowerFixedLengthVectorSelectToSVE()
21548 EVT VT = Op.getValueType(); in LowerFixedLengthBitcastToSVE() local
21549 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthBitcastToSVE()
21555 return convertFromScalableVector(DAG, VT, Op); in LowerFixedLengthBitcastToSVE()
21568 EVT VT = Op.getValueType(); in LowerFixedLengthConcatVectorsToSVE() local
21578 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops); in LowerFixedLengthConcatVectorsToSVE()
21581 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthConcatVectorsToSVE()
21589 return convertFromScalableVector(DAG, VT, Op); in LowerFixedLengthConcatVectorsToSVE()
21595 EVT VT = Op.getValueType(); in LowerFixedLengthFPExtendToSVE() local
21596 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthFPExtendToSVE()
21600 SDValue Pg = getPredicateForVector(DAG, DL, VT); in LowerFixedLengthFPExtendToSVE()
21602 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthFPExtendToSVE()
21607 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPExtendToSVE()
21614 return convertFromScalableVector(DAG, VT, Val); in LowerFixedLengthFPExtendToSVE()
21620 EVT VT = Op.getValueType(); in LowerFixedLengthFPRoundToSVE() local
21621 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthFPRoundToSVE()
21628 VT.getVectorElementType()); in LowerFixedLengthFPRoundToSVE()
21637 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPRoundToSVE()
21638 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthFPRoundToSVE()
21644 EVT VT = Op.getValueType(); in LowerFixedLengthIntToFPToSVE() local
21645 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthIntToFPToSVE()
21654 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthIntToFPToSVE()
21659 SDValue Pg = getPredicateForVector(DAG, DL, VT); in LowerFixedLengthIntToFPToSVE()
21662 VT.changeTypeToInteger(), Val); in LowerFixedLengthIntToFPToSVE()
21670 return convertFromScalableVector(DAG, VT, Val); in LowerFixedLengthIntToFPToSVE()
21681 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthIntToFPToSVE()
21682 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthIntToFPToSVE()
21689 EVT VT = Op.getValueType(); in LowerFixedLengthFPToIntToSVE() local
21690 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthFPToIntToSVE()
21699 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthFPToIntToSVE()
21706 SDValue Pg = getPredicateForVector(DAG, DL, VT); in LowerFixedLengthFPToIntToSVE()
21709 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21715 return convertFromScalableVector(DAG, VT, Val); in LowerFixedLengthFPToIntToSVE()
21726 return DAG.getNode(ISD::TRUNCATE, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21732 EVT VT = Op.getValueType(); in LowerFixedLengthVECTOR_SHUFFLEToSVE() local
21733 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!"); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21742 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21748 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm) && in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21749 Imm == VT.getVectorNumElements() - 1) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21753 EVT ScalarTy = VT.getVectorElementType(); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21758 DAG.getConstant(VT.getVectorNumElements() - 1, DL, MVT::i64)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21760 return convertFromScalableVector(DAG, VT, Op); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21764 if (isREVMask(ShuffleMask, VT, LaneSize)) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21768 unsigned EltSz = VT.getScalarSizeInBits(); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21779 return convertFromScalableVector(DAG, VT, Op); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21784 if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult == 0) in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21786 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21788 if (isTRNMask(ShuffleMask, VT, WhichResult)) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21791 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21794 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult == 0) in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21796 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21798 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21801 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21824 if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21827 return convertFromScalableVector(DAG, VT, Op); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21830 if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult != 0) in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21832 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21834 if (isUZPMask(ShuffleMask, VT, WhichResult)) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21837 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21840 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0) in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21842 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21844 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21847 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21854 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op, in getSVESafeBitCast() argument
21859 assert(VT.isScalableVector() && isTypeLegal(VT) && in getSVESafeBitCast()
21862 assert(VT.getVectorElementType() != MVT::i1 && in getSVESafeBitCast()
21866 if (InVT == VT) in getSVESafeBitCast()
21869 EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType()); in getSVESafeBitCast()
21879 assert((VT.getVectorElementCount() == InVT.getVectorElementCount() || in getSVESafeBitCast()
21880 VT == PackedVT || InVT == PackedInVT) && in getSVESafeBitCast()
21890 if (VT != PackedVT) in getSVESafeBitCast()
21891 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op); in getSVESafeBitCast()
21901 EVT AArch64TargetLowering::getPromotedVTForPredicate(EVT VT) const { in getPromotedVTForPredicate()
21902 return ::getPromotedVTForPredicate(VT); in getPromotedVTForPredicate()