Lines Matching refs:VT
283 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
284 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
285 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
297 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
298 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
299 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
506 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering() local
507 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
508 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
509 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
510 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
512 for (MVT VT : MVT::integer_valuetypes()) in AArch64TargetLowering() local
513 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
699 for (MVT VT : MVT::integer_valuetypes()) { in AArch64TargetLowering() local
700 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in AArch64TargetLowering()
701 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in AArch64TargetLowering()
702 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in AArch64TargetLowering()
703 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in AArch64TargetLowering()
704 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in AArch64TargetLowering()
706 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering() local
707 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in AArch64TargetLowering()
708 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in AArch64TargetLowering()
715 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
716 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
718 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) { in AArch64TargetLowering()
719 setOperationAction(ISD::MULHS, VT, Legal); in AArch64TargetLowering()
720 setOperationAction(ISD::MULHU, VT, Legal); in AArch64TargetLowering()
722 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
723 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
725 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
726 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
728 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
731 setTruncStoreAction(VT, InnerVT, Expand); in AArch64TargetLowering()
732 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
733 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
734 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
754 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) { in addTypeForNEON() argument
755 assert(VT.isVector() && "VT should be a vector type"); in addTypeForNEON()
757 if (VT.isFloatingPoint()) { in addTypeForNEON()
758 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT(); in addTypeForNEON()
759 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo); in addTypeForNEON()
760 setOperationPromotedToType(ISD::STORE, VT, PromoteTo); in addTypeForNEON()
764 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { in addTypeForNEON()
765 setOperationAction(ISD::FSIN, VT, Expand); in addTypeForNEON()
766 setOperationAction(ISD::FCOS, VT, Expand); in addTypeForNEON()
767 setOperationAction(ISD::FPOW, VT, Expand); in addTypeForNEON()
768 setOperationAction(ISD::FLOG, VT, Expand); in addTypeForNEON()
769 setOperationAction(ISD::FLOG2, VT, Expand); in addTypeForNEON()
770 setOperationAction(ISD::FLOG10, VT, Expand); in addTypeForNEON()
771 setOperationAction(ISD::FEXP, VT, Expand); in addTypeForNEON()
772 setOperationAction(ISD::FEXP2, VT, Expand); in addTypeForNEON()
775 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in addTypeForNEON()
778 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
779 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
780 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
781 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
782 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForNEON()
783 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
784 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
785 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
786 setOperationAction(ISD::AND, VT, Custom); in addTypeForNEON()
787 setOperationAction(ISD::OR, VT, Custom); in addTypeForNEON()
788 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
789 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
791 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
792 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
793 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
795 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in addTypeForNEON()
798 if (VT != MVT::v8i8 && VT != MVT::v16i8) in addTypeForNEON()
799 setOperationAction(ISD::CTPOP, VT, Custom); in addTypeForNEON()
801 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
802 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
803 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
804 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
805 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
807 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForNEON()
808 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForNEON()
810 if (!VT.isFloatingPoint()) in addTypeForNEON()
811 setOperationAction(ISD::ABS, VT, Legal); in addTypeForNEON()
814 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) in addTypeForNEON()
816 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
819 if (VT.isFloatingPoint() && in addTypeForNEON()
820 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16())) in addTypeForNEON()
823 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
828 setIndexedLoadAction(im, VT, Legal); in addTypeForNEON()
829 setIndexedStoreAction(im, VT, Legal); in addTypeForNEON()
834 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON() argument
835 addRegisterClass(VT, &AArch64::FPR64RegClass); in addDRTypeForNEON()
836 addTypeForNEON(VT, MVT::v2i32); in addDRTypeForNEON()
839 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON() argument
840 addRegisterClass(VT, &AArch64::FPR128RegClass); in addQRTypeForNEON()
841 addTypeForNEON(VT, MVT::v4i32); in addQRTypeForNEON()
845 EVT VT) const { in getSetCCResultType()
846 if (!VT.isVector()) in getSetCCResultType()
848 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
925 EVT VT = Op.getValueType(); in optimizeLogicalImm() local
932 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0), in optimizeLogicalImm()
933 TLO.DAG.getConstant(NewImm, DL, VT)); in optimizeLogicalImm()
938 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT); in optimizeLogicalImm()
940 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0); in optimizeLogicalImm()
955 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
956 if (VT.isVector()) in targetShrinkDemandedConstant()
959 unsigned Size = VT.getSizeInBits(); in targetShrinkDemandedConstant()
1012 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode() local
1013 unsigned MemBits = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
1032 MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); in computeKnownBitsForTargetNode() local
1034 if (VT == MVT::v8i8 || VT == MVT::v16i8) { in computeKnownBitsForTargetNode()
1038 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { in computeKnownBitsForTargetNode()
1055 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, in allowsMisalignedMemoryAccesses() argument
1064 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 || in allowsMisalignedMemoryAccesses()
1075 VT == MVT::v2i64; in allowsMisalignedMemoryAccesses()
1501 EVT VT = LHS.getValueType(); in emitComparison() local
1505 if (VT.isFloatingPoint()) { in emitComparison()
1506 assert(VT != MVT::f128); in emitComparison()
1507 if (VT == MVT::f16 && !FullFP16) { in emitComparison()
1510 VT = MVT::f32; in emitComparison()
1512 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); in emitComparison()
1540 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS) in emitComparison()
1859 EVT VT = Op.getValueType(); in getCmpOperandFoldingProfit() local
1860 if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63)) in getCmpOperandFoldingProfit()
1871 EVT VT = RHS.getValueType(); in getAArch64Cmp() local
1880 if ((VT == MVT::i32 && C != 0x80000000 && in getAArch64Cmp()
1882 (VT == MVT::i64 && C != 0x80000000ULL && in getAArch64Cmp()
1885 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1886 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1891 if ((VT == MVT::i32 && C != 0 && in getAArch64Cmp()
1893 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { in getAArch64Cmp()
1895 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1896 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1901 if ((VT == MVT::i32 && C != INT32_MAX && in getAArch64Cmp()
1903 (VT == MVT::i64 && C != INT64_MAX && in getAArch64Cmp()
1906 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1907 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1912 if ((VT == MVT::i32 && C != UINT32_MAX && in getAArch64Cmp()
1914 (VT == MVT::i64 && C != UINT64_MAX && in getAArch64Cmp()
1917 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1918 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
2209 EVT VT = Op.getValueType(); in LowerADDC_ADDE_SUBC_SUBE() local
2212 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDC_ADDE_SUBC_SUBE()
2215 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
2334 EVT VT = Op.getValueType(); in LowerVectorFP_TO_INT() local
2346 if (VT.getSizeInBits() < InVT.getSizeInBits()) { in LowerVectorFP_TO_INT()
2351 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
2354 if (VT.getSizeInBits() > InVT.getSizeInBits()) { in LowerVectorFP_TO_INT()
2357 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), in LowerVectorFP_TO_INT()
2358 VT.getVectorNumElements()); in LowerVectorFP_TO_INT()
2360 return DAG.getNode(Op.getOpcode(), dl, VT, Ext); in LowerVectorFP_TO_INT()
2400 EVT VT = Op.getValueType(); in LowerVectorINT_TO_FP() local
2405 if (VT.getSizeInBits() < InVT.getSizeInBits()) { in LowerVectorINT_TO_FP()
2410 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
2413 if (VT.getSizeInBits() > InVT.getSizeInBits()) { in LowerVectorINT_TO_FP()
2416 EVT CastVT = VT.changeVectorElementTypeToInteger(); in LowerVectorINT_TO_FP()
2418 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerVectorINT_TO_FP()
2542 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR() local
2549 unsigned EltSize = VT.getScalarSizeInBits(); in isExtendedBUILD_VECTOR()
2574 EVT VT = N->getValueType(0); in skipExtensionForVectorMULL() local
2576 unsigned EltSize = VT.getScalarSizeInBits() / 2; in skipExtensionForVectorMULL()
2577 unsigned NumElts = VT.getVectorNumElements(); in skipExtensionForVectorMULL()
2645 EVT VT = Op.getValueType(); in LowerMUL() local
2646 assert(VT.is128BitVector() && VT.isInteger() && in LowerMUL()
2678 if (VT == MVT::v2i64) in LowerMUL()
2696 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
2704 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
2705 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
2707 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
2779 EVT VT, EVT MemVT, in LowerTruncateVectorStore() argument
2781 assert(VT.isVector() && "VT should be a vector type"); in LowerTruncateVectorStore()
2782 assert(MemVT == MVT::v4i8 && VT == MVT::v4i16); in LowerTruncateVectorStore()
2820 EVT VT = Value.getValueType(); in LowerSTORE() local
2823 assert (VT.isVector() && "Can only custom lower vector store types"); in LowerSTORE()
2833 return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG); in LowerSTORE()
3034 MVT ValVT = Ins[i].VT; in LowerFormalArguments()
3119 assert(RegVT == Ins[i].VT && "incorrect register location selected"); in LowerFormalArguments()
3574 MVT ArgVT = Outs[i].VT; in LowerCall()
3591 MVT ValVT = Outs[i].VT; in LowerCall()
3662 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT); in LowerCall()
3704 Outs[0].VT == MVT::i64) { in LowerCall()
3707 assert(!Ins.empty() && Ins[0].VT == MVT::i64 && in LowerCall()
4515 EVT VT = Op.getValueType(); in LowerFCOPYSIGN() local
4522 if (SrcVT.bitsLT(VT)) in LowerFCOPYSIGN()
4523 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
4524 else if (SrcVT.bitsGT(VT)) in LowerFCOPYSIGN()
4525 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
4532 if (!VT.isVector()) { in LowerFCOPYSIGN()
4543 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) { in LowerFCOPYSIGN()
4544 VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32); in LowerFCOPYSIGN()
4547 } else if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
4556 } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) { in LowerFCOPYSIGN()
4557 VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16); in LowerFCOPYSIGN()
4568 if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
4577 if (VT == MVT::f16) in LowerFCOPYSIGN()
4578 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel); in LowerFCOPYSIGN()
4579 if (VT == MVT::f32) in LowerFCOPYSIGN()
4580 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel); in LowerFCOPYSIGN()
4581 else if (VT == MVT::f64) in LowerFCOPYSIGN()
4582 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel); in LowerFCOPYSIGN()
4584 return DAG.getNode(ISD::BITCAST, DL, VT, Sel); in LowerFCOPYSIGN()
4605 EVT VT = Op.getValueType(); in LowerCTPOP() local
4607 if (VT == MVT::i32 || VT == MVT::i64) { in LowerCTPOP()
4608 if (VT == MVT::i32) in LowerCTPOP()
4617 if (VT == MVT::i64) in LowerCTPOP()
4622 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || in LowerCTPOP()
4623 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && in LowerCTPOP()
4626 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; in LowerCTPOP()
4632 unsigned NumElts = VT.is64BitVector() ? 8 : 16; in LowerCTPOP()
4633 while (EltSize != VT.getScalarSizeInBits()) { in LowerCTPOP()
4656 EVT VT = Op.getValueType(); in LowerSETCC() local
4657 SDValue TVal = DAG.getConstant(1, dl, VT); in LowerSETCC()
4658 SDValue FVal = DAG.getConstant(0, dl, VT); in LowerSETCC()
4681 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); in LowerSETCC()
4701 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp); in LowerSETCC()
4711 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSETCC()
4714 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSETCC()
4857 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
4858 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); in LowerSELECT_CC()
4865 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
4893 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSELECT_CC()
4899 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSELECT_CC()
5146 EVT VT = Op.getValueType(); in LowerVAARG() local
5164 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); in LowerVAARG()
5171 if (VT.isInteger() && !VT.isVector()) in LowerVAARG()
5174 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { in LowerVAARG()
5192 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
5199 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo()); in LowerVAARG()
5207 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
5211 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT); in LowerFRAMEADDR()
5213 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
5222 EVT VT = getPointerTy(DAG.getDataLayout()); in LowerSPONENTRY() local
5225 return DAG.getFrameIndex(FI, VT); in LowerSPONENTRY()
5230 unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT, in getRegisterByName() argument
5282 EVT VT = Op.getValueType(); in LowerADDROFRETURNADDR() local
5286 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT); in LowerADDROFRETURNADDR()
5289 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset); in LowerADDROFRETURNADDR()
5298 EVT VT = Op.getValueType(); in LowerRETURNADDR() local
5304 return DAG.getLoad(VT, DL, DAG.getEntryNode(), in LowerRETURNADDR()
5305 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in LowerRETURNADDR()
5311 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); in LowerRETURNADDR()
5319 EVT VT = Op.getValueType(); in LowerShiftRightParts() local
5320 unsigned VTBits = VT.getSizeInBits(); in LowerShiftRightParts()
5331 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
5339 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftRightParts()
5345 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
5347 DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo); in LowerShiftRightParts()
5352 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); in LowerShiftRightParts()
5353 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, in LowerShiftRightParts()
5358 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); in LowerShiftRightParts()
5361 ? DAG.getNode(Opc, dl, VT, ShOpHi, in LowerShiftRightParts()
5363 : DAG.getConstant(0, dl, VT); in LowerShiftRightParts()
5364 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, in LowerShiftRightParts()
5376 EVT VT = Op.getValueType(); in LowerShiftLeftParts() local
5377 unsigned VTBits = VT.getSizeInBits(); in LowerShiftLeftParts()
5386 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
5394 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftLeftParts()
5399 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
5401 DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi); in LowerShiftLeftParts()
5403 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
5408 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, in LowerShiftLeftParts()
5413 SDValue LoForBigShift = DAG.getConstant(0, dl, VT); in LowerShiftLeftParts()
5414 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
5415 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, in LowerShiftLeftParts()
5429 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { in isFPImmLegal()
5432 if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32 || in isFPImmLegal()
5433 (VT == MVT::f16 && Subtarget->hasFullFP16()))) { in isFPImmLegal()
5434 LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() << " imm value: 0\n"); in isFPImmLegal()
5442 if (VT == MVT::f64) in isFPImmLegal()
5444 else if (VT == MVT::f32) in isFPImmLegal()
5446 else if (VT == MVT::f16 && Subtarget->hasFullFP16()) in isFPImmLegal()
5450 LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() in isFPImmLegal()
5455 LLVM_DEBUG(dbgs() << "Illegal " << VT.getEVTString() in isFPImmLegal()
5467 EVT VT = Operand.getValueType(); in getEstimate() local
5469 (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 || in getEstimate()
5470 VT == MVT::f32 || VT == MVT::v1f32 || in getEstimate()
5471 VT == MVT::v2f32 || VT == MVT::v4f32)) { in getEstimate()
5478 ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2; in getEstimate()
5480 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); in getEstimate()
5496 EVT VT = Operand.getValueType(); in getSqrtEstimate() local
5504 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate, in getSqrtEstimate()
5506 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags); in getSqrtEstimate()
5507 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getSqrtEstimate()
5511 VT); in getSqrtEstimate()
5512 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in getSqrtEstimate()
5515 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags); in getSqrtEstimate()
5517 Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, in getSqrtEstimate()
5518 VT, Eq, Operand, Estimate); in getSqrtEstimate()
5535 EVT VT = Operand.getValueType(); in getRecipEstimate() local
5543 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand, in getRecipEstimate()
5545 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getRecipEstimate()
5660 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
5664 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
5670 if (VT.getSizeInBits() == 16) in getRegForInlineAsmConstraint()
5672 if (VT.getSizeInBits() == 32) in getRegForInlineAsmConstraint()
5674 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
5676 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
5684 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
5695 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
5708 if (VT != MVT::Other && VT.getSizeInBits() == 64) { in getRegForInlineAsmConstraint()
5888 EVT VT = V64Reg.getValueType(); in WidenVector() local
5889 unsigned NarrowSize = VT.getVectorNumElements(); in WidenVector()
5890 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in WidenVector()
5908 EVT VT = V128Reg.getValueType(); in NarrowVector() local
5909 unsigned WideSize = VT.getVectorNumElements(); in NarrowVector()
5910 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
5924 EVT VT = Op.getValueType(); in ReconstructShuffle() local
5925 unsigned NumElts = VT.getVectorNumElements(); in ReconstructShuffle()
5987 EVT SmallestEltTy = VT.getVectorElementType(); in ReconstructShuffle()
5995 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
5996 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
6005 if (SrcVT.getSizeInBits() == VT.getSizeInBits()) in ReconstructShuffle()
6011 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); in ReconstructShuffle()
6014 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { in ReconstructShuffle()
6015 assert(2 * SrcVT.getSizeInBits() == VT.getSizeInBits()); in ReconstructShuffle()
6024 assert(SrcVT.getSizeInBits() == 2 * VT.getSizeInBits()); in ReconstructShuffle()
6081 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { in ReconstructShuffle()
6094 std::min(OrigEltTy.getSizeInBits(), VT.getScalarSizeInBits()); in ReconstructShuffle()
6119 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
6129 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { in isSingletonEXTMask() argument
6130 unsigned NumElts = VT.getVectorNumElements(); in isSingletonEXTMask()
6160 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT, in isEXTMask() argument
6166 unsigned NumElts = VT.getVectorNumElements(); in isEXTMask()
6201 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { in isREVMask() argument
6205 unsigned EltSz = VT.getScalarSizeInBits(); in isREVMask()
6209 unsigned NumElts = VT.getVectorNumElements(); in isREVMask()
6228 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIPMask() argument
6229 unsigned NumElts = VT.getVectorNumElements(); in isZIPMask()
6242 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZPMask() argument
6243 unsigned NumElts = VT.getVectorNumElements(); in isUZPMask()
6255 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRNMask() argument
6256 unsigned NumElts = VT.getVectorNumElements(); in isTRNMask()
6269 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIP_v_undef_Mask() argument
6270 unsigned NumElts = VT.getVectorNumElements(); in isZIP_v_undef_Mask()
6286 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZP_v_undef_Mask() argument
6287 unsigned Half = VT.getVectorNumElements() / 2; in isUZP_v_undef_Mask()
6305 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRN_v_undef_Mask() argument
6306 unsigned NumElts = VT.getVectorNumElements(); in isTRN_v_undef_Mask()
6355 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) { in isConcatMask() argument
6356 if (VT.getSizeInBits() != 128) in isConcatMask()
6359 unsigned NumElts = VT.getVectorNumElements(); in isConcatMask()
6377 EVT VT = Op.getValueType(); in tryFormConcatFromShuffle() local
6382 if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() || in tryFormConcatFromShuffle()
6383 VT.getVectorElementType() != V1.getValueType().getVectorElementType()) in tryFormConcatFromShuffle()
6388 if (!isConcatMask(Mask, VT, SplitV0)) in tryFormConcatFromShuffle()
6391 EVT CastVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in tryFormConcatFromShuffle()
6392 VT.getVectorNumElements() / 2); in tryFormConcatFromShuffle()
6401 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
6441 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
6448 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
6449 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
6450 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
6452 if (VT.getVectorElementType() == MVT::i16 || in GeneratePerfectShuffle()
6453 VT.getVectorElementType() == MVT::f16) in GeneratePerfectShuffle()
6454 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
6456 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
6457 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
6462 EVT EltTy = VT.getVectorElementType(); in GeneratePerfectShuffle()
6475 if (VT.getSizeInBits() == 64) in GeneratePerfectShuffle()
6478 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane); in GeneratePerfectShuffle()
6484 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS, in GeneratePerfectShuffle()
6488 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6491 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6494 return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6497 return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6500 return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6503 return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
6586 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE() local
6612 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane)); in LowerVECTOR_SHUFFLE()
6624 unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2; in LowerVECTOR_SHUFFLE()
6625 Lane -= Idx * VT.getVectorNumElements() / 2; in LowerVECTOR_SHUFFLE()
6627 } else if (VT.getSizeInBits() == 64) in LowerVECTOR_SHUFFLE()
6630 return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64)); in LowerVECTOR_SHUFFLE()
6633 if (isREVMask(ShuffleMask, VT, 64)) in LowerVECTOR_SHUFFLE()
6635 if (isREVMask(ShuffleMask, VT, 32)) in LowerVECTOR_SHUFFLE()
6637 if (isREVMask(ShuffleMask, VT, 16)) in LowerVECTOR_SHUFFLE()
6642 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) { in LowerVECTOR_SHUFFLE()
6648 } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
6655 if (isZIPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6659 if (isUZPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6663 if (isTRNMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6668 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6672 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6676 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
6695 SrcLane -= VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
6699 EVT ScalarVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE()
6705 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
6712 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
6737 EVT VT = BVN->getValueType(0); in resolveBuildVector() local
6742 unsigned NumSplats = VT.getSizeInBits() / SplatBitSize; in resolveBuildVector()
6747 CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
6748 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
6762 EVT VT = Op.getValueType(); in tryAdvSIMDModImm64() local
6763 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64; in tryAdvSIMDModImm64()
6771 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm64()
6784 EVT VT = Op.getValueType(); in tryAdvSIMDModImm32() local
6785 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in tryAdvSIMDModImm32()
6819 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm32()
6832 EVT VT = Op.getValueType(); in tryAdvSIMDModImm16() local
6833 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in tryAdvSIMDModImm16()
6859 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm16()
6871 EVT VT = Op.getValueType(); in tryAdvSIMDModImm321s() local
6872 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in tryAdvSIMDModImm321s()
6890 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm321s()
6902 EVT VT = Op.getValueType(); in tryAdvSIMDModImm8() local
6903 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; in tryAdvSIMDModImm8()
6911 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm8()
6923 EVT VT = Op.getValueType(); in tryAdvSIMDModImmFP() local
6924 bool isWide = (VT.getSizeInBits() == 128); in tryAdvSIMDModImmFP()
6942 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImmFP()
6952 EVT VT = Op.getValueType(); in LowerVectorAND() local
6964 APInt DefBits(VT.getSizeInBits(), 0); in LowerVectorAND()
6965 APInt UndefBits(VT.getSizeInBits(), 0); in LowerVectorAND()
7000 EVT VT = Bvec->getValueType(0); in isAllConstantBuildVector() local
7001 unsigned NumElts = VT.getVectorNumElements(); in isAllConstantBuildVector()
7028 EVT VT = N->getValueType(0); in tryLowerToSLI() local
7030 if (!VT.isVector()) in tryLowerToSLI()
7062 unsigned ElemSizeInBits = VT.getScalarSizeInBits(); in tryLowerToSLI()
7075 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in tryLowerToSLI()
7096 EVT VT = Op.getValueType(); in LowerVectorOR() local
7109 APInt DefBits(VT.getSizeInBits(), 0); in LowerVectorOR()
7110 APInt UndefBits(VT.getSizeInBits(), 0); in LowerVectorOR()
7137 EVT VT = Op.getValueType(); in NormalizeBuildVector() local
7138 EVT EltTy= VT.getVectorElementType(); in NormalizeBuildVector()
7161 return DAG.getBuildVector(VT, dl, Ops); in NormalizeBuildVector()
7165 EVT VT = Op.getValueType(); in ConstantBuildVector() local
7167 APInt DefBits(VT.getSizeInBits(), 0); in ConstantBuildVector()
7168 APInt UndefBits(VT.getSizeInBits(), 0); in ConstantBuildVector()
7207 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR() local
7211 if (VT.isInteger()) { in LowerBUILD_VECTOR()
7219 unsigned BitSize = VT.getVectorElementType().getSizeInBits(); in LowerBUILD_VECTOR()
7243 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
7280 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
7289 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
7310 if (VT.getVectorElementType() != in LowerBUILD_VECTOR()
7338 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
7341 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
7345 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
7348 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
7358 Value.getValueType() != VT) { in LowerBUILD_VECTOR()
7361 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value); in LowerBUILD_VECTOR()
7375 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType()); in LowerBUILD_VECTOR()
7376 return DAG.getNode(Opcode, dl, VT, Value, Lane); in LowerBUILD_VECTOR()
7379 if (VT.getVectorElementType().isFloatingPoint()) { in LowerBUILD_VECTOR()
7381 EVT EltTy = VT.getVectorElementType(); in LowerBUILD_VECTOR()
7396 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
7406 SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue), in LowerBUILD_VECTOR()
7410 Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue); in LowerBUILD_VECTOR()
7421 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
7451 SDValue Vec = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
7467 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0); in LowerBUILD_VECTOR()
7477 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
7493 EVT VT = Op.getOperand(0).getValueType(); in LowerINSERT_VECTOR_ELT() local
7495 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerINSERT_VECTOR_ELT()
7500 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerINSERT_VECTOR_ELT()
7501 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerINSERT_VECTOR_ELT()
7502 VT == MVT::v8f16) in LowerINSERT_VECTOR_ELT()
7505 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerINSERT_VECTOR_ELT()
7506 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerINSERT_VECTOR_ELT()
7527 EVT VT = Op.getOperand(0).getValueType(); in LowerEXTRACT_VECTOR_ELT() local
7529 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerEXTRACT_VECTOR_ELT()
7534 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerEXTRACT_VECTOR_ELT()
7535 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerEXTRACT_VECTOR_ELT()
7536 VT == MVT::v8f16) in LowerEXTRACT_VECTOR_ELT()
7539 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerEXTRACT_VECTOR_ELT()
7540 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerEXTRACT_VECTOR_ELT()
7560 EVT VT = Op.getOperand(0).getValueType(); in LowerEXTRACT_SUBVECTOR() local
7563 if (!VT.isVector()) in LowerEXTRACT_SUBVECTOR()
7579 if (Size == 64 && Val * VT.getScalarSizeInBits() == 64) in LowerEXTRACT_SUBVECTOR()
7585 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { in isShuffleMaskLegal()
7586 if (VT.getVectorNumElements() == 4 && in isShuffleMaskLegal()
7587 (VT.is128BitVector() || VT.is64BitVector())) { in isShuffleMaskLegal()
7610 return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) || in isShuffleMaskLegal()
7611 isREVMask(M, VT, 32) || isREVMask(M, VT, 16) || in isShuffleMaskLegal()
7612 isEXTMask(M, VT, DummyBool, DummyUnsigned) || in isShuffleMaskLegal()
7614 isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
7615 isZIPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
7616 isTRN_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
7617 isUZP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
7618 isZIP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
7619 isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) || in isShuffleMaskLegal()
7620 isConcatMask(M, VT, VT.getSizeInBits() == 128)); in isShuffleMaskLegal()
7646 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { in isVShiftLImm() argument
7647 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftLImm()
7648 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftLImm()
7657 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) { in isVShiftRImm() argument
7658 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftRImm()
7659 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftRImm()
7667 EVT VT = Op.getValueType(); in LowerVectorSRA_SRL_SHL() local
7673 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVectorSRA_SRL_SHL()
7680 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) in LowerVectorSRA_SRL_SHL()
7681 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
7683 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
7690 if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) { in LowerVectorSRA_SRL_SHL()
7693 return DAG.getNode(Opc, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
7703 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1)); in LowerVectorSRA_SRL_SHL()
7705 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
7715 AArch64CC::CondCode CC, bool NoNans, EVT VT, in EmitVectorComparison() argument
7718 assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && in EmitVectorComparison()
7722 APInt CnstBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
7723 APInt UndefBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
7734 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
7736 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
7737 return DAG.getNode(AArch64ISD::NOT, dl, VT, Fcmeq); in EmitVectorComparison()
7741 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
7742 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
7745 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS); in EmitVectorComparison()
7746 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
7749 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS); in EmitVectorComparison()
7750 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
7753 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS); in EmitVectorComparison()
7754 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
7762 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); in EmitVectorComparison()
7763 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
7773 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
7775 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
7776 return DAG.getNode(AArch64ISD::NOT, dl, VT, Cmeq); in EmitVectorComparison()
7780 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
7781 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
7784 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS); in EmitVectorComparison()
7785 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
7788 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); in EmitVectorComparison()
7789 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
7792 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); in EmitVectorComparison()
7793 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
7795 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS); in EmitVectorComparison()
7797 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS); in EmitVectorComparison()
7800 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); in EmitVectorComparison()
7801 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
7803 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); in EmitVectorComparison()
7805 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS); in EmitVectorComparison()
7922 MVT VT = Op.getSimpleValueType(); in LowerATOMIC_LOAD_SUB() local
7925 RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS); in LowerATOMIC_LOAD_SUB()
7939 MVT VT = Op.getSimpleValueType(); in LowerATOMIC_LOAD_AND() local
7942 RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS); in LowerATOMIC_LOAD_AND()
7987 EVT VT = Node->getValueType(0); in LowerDYNAMIC_STACKALLOC() local
7995 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
7996 DAG.getConstant(-(uint64_t)Align, dl, VT)); in LowerDYNAMIC_STACKALLOC()
8010 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
8011 DAG.getConstant(-(uint64_t)Align, dl, VT)); in LowerDYNAMIC_STACKALLOC()
8192 EVT VT = getValueType(DL, User->getOperand(0)->getType()); in isProfitableToHoist() local
8194 return !(isFMAFasterThanFMulAndFAdd(VT) && in isProfitableToHoist()
8195 isOperationLegalOrCustom(ISD::FMA, VT) && in isProfitableToHoist()
8598 auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) { in getOptimalMemOpType() argument
8602 return allowsMisalignedMemoryAccesses(VT, 0, 1, &Fast) && Fast; in getOptimalMemOpType()
8712 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { in isFMAFasterThanFMulAndFAdd()
8713 VT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
8715 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
8718 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
8744 EVT VT = N->getValueType(0); in isDesirableToCommuteWithShift() local
8747 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
8795 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp() local
8796 if (!Subtarget->hasNEON() || !VT.isVector()) in foldVectorXorShiftIntoCmp()
8813 return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0)); in foldVectorXorShiftIntoCmp()
8818 EVT VT = N->getValueType(0); in performIntegerAbsCombine() local
8826 if (VT.isInteger() && N->getOpcode() == ISD::XOR && in performIntegerAbsCombine()
8830 if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) { in performIntegerAbsCombine()
8831 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in performIntegerAbsCombine()
8835 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
8836 N0.getOperand(0), DAG.getConstant(0, DL, VT)); in performIntegerAbsCombine()
8837 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg, in performIntegerAbsCombine()
8865 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
8866 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSDIVPow2()
8873 SDValue Zero = DAG.getConstant(0, DL, VT); in BuildSDIVPow2()
8874 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT); in BuildSDIVPow2()
8879 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
8880 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp); in BuildSDIVPow2()
8888 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
8896 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
8980 EVT VT = N->getValueType(0); in performMulCombine() local
8981 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0, in performMulCombine()
8986 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1); in performMulCombine()
8991 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); in performMulCombine()
8994 return DAG.getNode(ISD::SHL, DL, VT, Res, in performMulCombine()
9012 EVT VT = N->getValueType(0); in performVectorCompareAndMaskUnaryOpCombine() local
9013 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
9015 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits()) in performVectorCompareAndMaskUnaryOpCombine()
9033 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in performVectorCompareAndMaskUnaryOpCombine()
9038 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
9052 EVT VT = N->getValueType(0); in performIntToFpCombine() local
9053 if (VT != MVT::f32 && VT != MVT::f64) in performIntToFpCombine()
9057 if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits()) in performIntToFpCombine()
9068 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in performIntToFpCombine()
9078 return DAG.getNode(Opcode, SDLoc(N), VT, Load); in performIntToFpCombine()
9253 EVT VT = N->getValueType(0); in tryCombineToEXTR() local
9257 if (VT != MVT::i32 && VT != MVT::i64) in tryCombineToEXTR()
9277 if (ShiftLHS + ShiftRHS != VT.getSizeInBits()) in tryCombineToEXTR()
9285 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS, in tryCombineToEXTR()
9291 EVT VT = N->getValueType(0); in tryCombineToBSL() local
9295 if (!VT.isVector()) in tryCombineToBSL()
9308 unsigned Bits = VT.getScalarSizeInBits(); in tryCombineToBSL()
9318 for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) { in tryCombineToBSL()
9329 return DAG.getNode(AArch64ISD::BSL, DL, VT, SDValue(BVN0, 0), in tryCombineToBSL()
9340 EVT VT = N->getValueType(0); in performORCombine() local
9342 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in performORCombine()
9357 EVT VT = N->getValueType(0); in performSRLCombine() local
9358 if (VT != MVT::i32 && VT != MVT::i64) in performSRLCombine()
9371 if (VT == MVT::i32 && ShiftAmt == 16 && in performSRLCombine()
9373 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); in performSRLCombine()
9374 if (VT == MVT::i64 && ShiftAmt == 32 && in performSRLCombine()
9376 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); in performSRLCombine()
9398 EVT VT = N->getValueType(0); in performBitcastCombine() local
9399 if (!VT.isVector()) in performBitcastCombine()
9401 if (VT.getSimpleVT().getSizeInBits() != 64) in performBitcastCombine()
9429 SVT.getVectorNumElements() != VT.getVectorNumElements() * 2) in performBitcastCombine()
9438 unsigned NumElements = VT.getVectorNumElements(); in performBitcastCombine()
9441 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx); in performBitcastCombine()
9444 return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT, in performBitcastCombine()
9454 EVT VT = N->getValueType(0); in performConcatVectorsCombine() local
9477 N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) { in performConcatVectorsCombine()
9482 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
9498 if (N0 == N1 && VT.getVectorNumElements() == 2) { in performConcatVectorsCombine()
9499 assert(VT.getScalarSizeInBits() == 64); in performConcatVectorsCombine()
9500 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), in performConcatVectorsCombine()
9526 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
9760 EVT VT = Op->getValueType(0); in performSetccAddFolding() local
9761 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
9762 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp); in performSetccAddFolding()
9782 MVT VT = N->getSimpleValueType(0); in performAddSubLongCombine() local
9783 if (!VT.is128BitVector()) { in performAddSubLongCombine()
9806 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); in performAddSubLongCombine()
9812 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS); in performAddSubLongCombine()
9815 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS); in performAddSubLongCombine()
10151 EVT VT = StVal.getValueType(); in replaceZeroVectorStore() local
10155 int NumVecElts = VT.getVectorNumElements(); in replaceZeroVectorStore()
10157 VT.getVectorElementType().getSizeInBits() == 64) || in replaceZeroVectorStore()
10159 VT.getVectorElementType().getSizeInBits() == 32))) in replaceZeroVectorStore()
10195 if (VT.getVectorElementType().getSizeInBits() == 32) { in replaceZeroVectorStore()
10214 EVT VT = StVal.getValueType(); in replaceSplatVectorStore() local
10218 if (VT.isFloatingPoint()) in replaceSplatVectorStore()
10222 unsigned NumVecElts = VT.getVectorNumElements(); in replaceSplatVectorStore()
10274 EVT VT = StVal.getValueType(); in splitStores() local
10275 if (!VT.isVector()) in splitStores()
10297 if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64) in splitStores()
10305 if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 || in splitStores()
10316 unsigned NumElts = VT.getVectorNumElements() / 2; in splitStores()
10319 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts); in splitStores()
10344 EVT VT = N->getValueType(0); in performPostLD1Combine() local
10357 if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements()) in performPostLD1Combine()
10364 if (MemVT != VT.getVectorElementType()) in performPostLD1Combine()
10391 unsigned NumBytes = VT.getScalarSizeInBits() / 8; in performPostLD1Combine()
10418 EVT Tys[3] = { VT, MVT::i64, MVT::Other }; in performPostLD1Combine()
11346 EVT VT; in getPreIndexedAddressParts() local
11349 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
11352 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
11367 EVT VT; in getPostIndexedAddressParts() local
11370 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
11373 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
11574 AArch64TargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
11577 if (VT == MVT::v1i8 || VT == MVT::v1i16 || VT == MVT::v1i32 || in getPreferredVectorAction()
11578 VT == MVT::v1f32) in getPreferredVectorAction()
11581 return TargetLoweringBase::getPreferredVectorAction(VT); in getPreferredVectorAction()
11843 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { in isIntDivCheap() argument
11853 return OptSize && !VT.isVector(); in isIntDivCheap()
11856 bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const { in enableAggressiveFMAFusion()
11857 return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint(); in enableAggressiveFMAFusion()