Lines Matching refs:MVT
147 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectAddrModeIndexedUImm()
219 template<MVT::SimpleValueType VT>
224 template <MVT::SimpleValueType VT>
229 template <MVT::SimpleValueType VT, bool Invert = false>
234 template <MVT::SimpleValueType VT>
269 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32); in SelectCntImm()
285 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32); in SelectEXTImm()
295 Imm = CurDAG->getRegister(BaseReg + C, MVT::Other); in ImmToTile()
406 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
407 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
408 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
414 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
462 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64); in SelectInlineAsmMemoryOperand()
499 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32); in SelectArithImmed()
500 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32); in SelectArithImmed()
525 if (N.getValueType() == MVT::i32) in SelectNegArithImmed()
533 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val, in SelectNegArithImmed()
621 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32); in SelectShiftedRegister()
640 if (!IsLoadStore && SrcVT == MVT::i8) in getExtendTypeForNode()
642 else if (!IsLoadStore && SrcVT == MVT::i16) in getExtendTypeForNode()
644 else if (SrcVT == MVT::i32) in getExtendTypeForNode()
646 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
652 if (!IsLoadStore && SrcVT == MVT::i8) in getExtendTypeForNode()
654 else if (!IsLoadStore && SrcVT == MVT::i16) in getExtendTypeForNode()
656 else if (SrcVT == MVT::i32) in getExtendTypeForNode()
658 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
739 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64); in tryMLAV64LaneV128()
748 case MVT::v4i16: in tryMLAV64LaneV128()
751 case MVT::v8i16: in tryMLAV64LaneV128()
754 case MVT::v2i32: in tryMLAV64LaneV128()
757 case MVT::v4i32: in tryMLAV64LaneV128()
776 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64); in tryMULLV64LaneV128()
786 case MVT::v4i32: in tryMULLV64LaneV128()
789 case MVT::v2i64: in tryMULLV64LaneV128()
797 case MVT::v4i32: in tryMULLV64LaneV128()
800 case MVT::v2i64: in tryMULLV64LaneV128()
816 if (N.getValueType() == MVT::i32) in narrowIfNeeded()
820 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in narrowIfNeeded()
822 dl, MVT::i32, N, SubReg); in narrowIfNeeded()
836 Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32); in SelectRDVLImm()
893 MVT::i32); in SelectArithExtendedRegister()
917 MVT::i32); in SelectArithUXTXRegister()
955 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
975 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
990 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1001 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1016 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
1043 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexed()
1059 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
1087 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64); in SelectAddrModeUnscaled()
1096 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in Widen()
1098 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0); in Widen()
1100 TargetOpcode::INSERT_SUBREG, dl, MVT::i64, ImpDef, N, SubReg); in Widen()
1123 MVT::i32); in SelectExtendedSHL()
1126 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32); in SelectExtendedSHL()
1169 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1177 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1182 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32); in SelectAddrModeWRO()
1192 MVT::i32); in SelectAddrModeWRO()
1204 MVT::i32); in SelectAddrModeWRO()
1269 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops); in SelectAddrModeXRO()
1272 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV); in SelectAddrModeXRO()
1282 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1290 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1297 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1298 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1347 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32)); in createTuple()
1352 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32)); in createTuple()
1356 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); in createTuple()
1397 if (VT == MVT::i64) in tryIndexedLoad()
1399 else if (VT == MVT::i32) { in tryIndexedLoad()
1409 DstVT = MVT::i32; in tryIndexedLoad()
1411 } else if (VT == MVT::i16) { in tryIndexedLoad()
1413 if (DstVT == MVT::i64) in tryIndexedLoad()
1419 InsertTo64 = DstVT == MVT::i64; in tryIndexedLoad()
1422 DstVT = MVT::i32; in tryIndexedLoad()
1424 } else if (VT == MVT::i8) { in tryIndexedLoad()
1426 if (DstVT == MVT::i64) in tryIndexedLoad()
1432 InsertTo64 = DstVT == MVT::i64; in tryIndexedLoad()
1435 DstVT = MVT::i32; in tryIndexedLoad()
1437 } else if (VT == MVT::f16) { in tryIndexedLoad()
1439 } else if (VT == MVT::bf16) { in tryIndexedLoad()
1441 } else if (VT == MVT::f32) { in tryIndexedLoad()
1443 } else if (VT == MVT::f64 || VT.is64BitVector()) { in tryIndexedLoad()
1454 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64); in tryIndexedLoad()
1456 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT, in tryIndexedLoad()
1457 MVT::Other, Ops); in tryIndexedLoad()
1466 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in tryIndexedLoad()
1469 AArch64::SUBREG_TO_REG, dl, MVT::i64, in tryIndexedLoad()
1470 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal, in tryIndexedLoad()
1491 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectLoad()
1521 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostLoad()
1522 MVT::Untyped, MVT::Other}; in SelectPostLoad()
1580 CurDAG->getTargetConstant(0, DL, MVT::i64), Scale); in SelectPredicatedLoad()
1586 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectPredicatedLoad()
1634 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale); in SelectPredicatedStore()
1655 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeFrameIndexSVE()
1666 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostStore()
1667 MVT::Other}; // Type for the Chain in SelectPostStore()
1695 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in operator ()()
1696 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize); in operator ()()
1711 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
1712 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2); in NarrowVector()
1733 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectLoadLane()
1738 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectLoadLane()
1772 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostLoadLane()
1773 RegSeq->getValueType(0), MVT::Other}; in SelectPostLoadLane()
1780 MVT::i64), // Lane Number in SelectPostLoadLane()
1830 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectStoreLane()
1832 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in SelectStoreLane()
1856 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostStoreLane()
1857 MVT::Other}; in SelectPostStoreLane()
1862 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectPostStoreLane()
1888 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromAnd()
1916 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND && in isBitfieldExtractOpFromAnd()
1923 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE && in isBitfieldExtractOpFromAnd()
1933 ClampMSB = (VT == MVT::i32); in isBitfieldExtractOpFromAnd()
1954 MSB = SrlImm + (VT == MVT::i32 ? countTrailingOnes<uint32_t>(AndImm) in isBitfieldExtractOpFromAnd()
1964 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri; in isBitfieldExtractOpFromAnd()
1975 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromSExtInReg()
1994 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri; in isBitfieldExtractOpFromSExtInReg()
2034 if (N->getValueType(0) == MVT::i32) in isSeveralBitsExtractOpFromShr()
2058 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromShr()
2070 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL && in isBitfieldExtractOpFromShr()
2079 assert(VT == MVT::i64 && "the promoted type should be i64"); in isBitfieldExtractOpFromShr()
2107 if (VT == MVT::i32) in isBitfieldExtractOpFromShr()
2119 if (VT != MVT::i64 || NarrowVT != MVT::i32) in tryBitfieldExtractOpFromSExt()
2147 if ((VT != MVT::v2f64 || NarrowVT != MVT::v2f32) && in tryHighFPExt()
2148 (VT != MVT::v4f32 || NarrowVT != MVT::v4f16)) in tryHighFPExt()
2162 auto Opcode = VT == MVT::v2f64 ? AArch64::FCVTLv4i32 : AArch64::FCVTLv8i16; in tryHighFPExt()
2171 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64) in isBitfieldExtractOp()
2219 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) { in tryBitfieldExtractOp()
2220 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64), in tryBitfieldExtractOp()
2221 CurDAG->getTargetConstant(Imms, dl, MVT::i64)}; in tryBitfieldExtractOp()
2223 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64); in tryBitfieldExtractOp()
2224 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in tryBitfieldExtractOp()
2226 MVT::i32, SDValue(BFM, 0), SubReg)); in tryBitfieldExtractOp()
2242 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldDstMask()
2556 assert(VT == MVT::i32 || VT == MVT::i64); in isShiftedMask()
2557 if (VT == MVT::i32) in isShiftedMask()
2568 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOrAndImm()
2639 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm; in tryBitfieldInsertOpFromOrAndImm()
2647 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOrAndImm()
2657 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOr()
2702 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) || in tryBitfieldInsertOpFromOr()
2703 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32)) in tryBitfieldInsertOpFromOr()
2727 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand"); in tryBitfieldInsertOpFromOr()
2758 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
2791 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertOpFromOr()
2812 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
2847 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertInZeroOp()
2864 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertInZeroOp()
2877 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr; in tryShiftAmountMod()
2880 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr; in tryShiftAmountMod()
2883 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr; in tryShiftAmountMod()
2886 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr; in tryShiftAmountMod()
2894 if (VT == MVT::i32) { in tryShiftAmountMod()
2897 } else if (VT == MVT::i64) { in tryShiftAmountMod()
2929 if (SubVT == MVT::i32) { in tryShiftAmountMod()
2933 assert(SubVT == MVT::i64); in tryShiftAmountMod()
2949 if (SubVT == MVT::i32) { in tryShiftAmountMod()
2953 assert(SubVT == MVT::i64); in tryShiftAmountMod()
2980 if (VT == MVT::i32) in tryShiftAmountMod()
2982 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) { in tryShiftAmountMod()
2983 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32); in tryShiftAmountMod()
2986 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg); in tryShiftAmountMod()
3035 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32); in SelectCVTFixedPosOperand()
3083 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
3084 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryReadRegister()
3100 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
3101 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryReadRegister()
3108 AArch64::ADR, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
3109 CurDAG->getTargetConstant(0, DL, MVT::i32), in tryReadRegister()
3129 N, CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other, in tryWriteRegister()
3130 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3155 State, DL, MVT::Other, in tryWriteRegister()
3156 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3157 CurDAG->getTargetConstant(Immed, DL, MVT::i16), in tryWriteRegister()
3173 AArch64::MSR, DL, MVT::Other, in tryWriteRegister()
3174 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3190 if (MemTy == MVT::i8) in SelectCMP_SWAP()
3192 else if (MemTy == MVT::i16) in SelectCMP_SWAP()
3194 else if (MemTy == MVT::i32) in SelectCMP_SWAP()
3196 else if (MemTy == MVT::i64) in SelectCMP_SWAP()
3201 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32; in SelectCMP_SWAP()
3206 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops); in SelectCMP_SWAP()
3218 bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVEAddSubImm()
3230 case MVT::i8: in SelectSVEAddSubImm()
3232 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
3233 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubImm()
3235 case MVT::i16: in SelectSVEAddSubImm()
3236 case MVT::i32: in SelectSVEAddSubImm()
3237 case MVT::i64: in SelectSVEAddSubImm()
3240 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
3241 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubImm()
3246 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVEAddSubImm()
3247 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32); in SelectSVEAddSubImm()
3258 bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVECpyDupImm()
3270 case MVT::i8: in SelectSVECpyDupImm()
3272 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVECpyDupImm()
3273 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
3275 case MVT::i16: in SelectSVECpyDupImm()
3276 case MVT::i32: in SelectSVECpyDupImm()
3277 case MVT::i64: in SelectSVECpyDupImm()
3280 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVECpyDupImm()
3281 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
3286 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVECpyDupImm()
3287 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
3303 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32); in SelectSVESignedArithImm()
3310 bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) { in SelectSVEArithImm()
3315 case MVT::i8: in SelectSVEArithImm()
3318 case MVT::i16: in SelectSVEArithImm()
3321 case MVT::i32: in SelectSVEArithImm()
3324 case MVT::i64: in SelectSVEArithImm()
3331 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEArithImm()
3338 bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVELogicalImm()
3349 case MVT::i8: in SelectSVELogicalImm()
3355 case MVT::i16: in SelectSVELogicalImm()
3360 case MVT::i32: in SelectSVELogicalImm()
3364 case MVT::i64: in SelectSVELogicalImm()
3372 Imm = CurDAG->getTargetConstant(encoding, DL, MVT::i64); in SelectSVELogicalImm()
3401 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEShiftImm()
3431 AArch64::TAGPstack, DL, MVT::i64, in trySelectStackSlotTagP()
3432 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2), in trySelectStackSlotTagP()
3433 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in trySelectStackSlotTagP()
3449 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64, in SelectTagP()
3451 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64, in SelectTagP()
3454 AArch64::ADDG, DL, MVT::i64, in SelectTagP()
3455 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64), in SelectTagP()
3456 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in SelectTagP()
3473 auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32); in extractSubReg()
3477 auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32); in extractSubReg()
3481 auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in extractSubReg()
3499 auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32); in insertSubReg()
3505 auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32); in insertSubReg()
3511 auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in insertSubReg()
3641 if (VT == MVT::i32) { in Select()
3643 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32); in Select()
3646 } else if (VT == MVT::i64) { in Select()
3648 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64); in Select()
3664 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32), in Select()
3665 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) }; in Select()
3666 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops); in Select()
3682 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64, in Select()
3683 MVT::Other, MemAddr, Chain); in Select()
3705 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops); in Select()
3715 if (VT == MVT::v8i8) { in Select()
3718 } else if (VT == MVT::v16i8) { in Select()
3721 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3724 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3727 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3730 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3733 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3736 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3742 if (VT == MVT::v8i8) { in Select()
3745 } else if (VT == MVT::v16i8) { in Select()
3748 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3751 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3754 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3757 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3760 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3763 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3769 if (VT == MVT::v8i8) { in Select()
3772 } else if (VT == MVT::v16i8) { in Select()
3775 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3778 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3781 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3784 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3787 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3790 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3796 if (VT == MVT::v8i8) { in Select()
3799 } else if (VT == MVT::v16i8) { in Select()
3802 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3805 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3808 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3811 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3814 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3817 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3823 if (VT == MVT::v8i8) { in Select()
3826 } else if (VT == MVT::v16i8) { in Select()
3829 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3832 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3835 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3838 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3841 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3844 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3850 if (VT == MVT::v8i8) { in Select()
3853 } else if (VT == MVT::v16i8) { in Select()
3856 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3859 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3862 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3865 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3868 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3871 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3877 if (VT == MVT::v8i8) { in Select()
3880 } else if (VT == MVT::v16i8) { in Select()
3883 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3886 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3889 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3892 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3895 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3898 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3904 if (VT == MVT::v8i8) { in Select()
3907 } else if (VT == MVT::v16i8) { in Select()
3910 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3913 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3916 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3919 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3922 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3925 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3931 if (VT == MVT::v8i8) { in Select()
3934 } else if (VT == MVT::v16i8) { in Select()
3937 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3940 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3943 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3946 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3949 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3952 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3958 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3961 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3962 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
3965 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
3966 VT == MVT::v2f32) { in Select()
3969 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
3970 VT == MVT::v1f64) { in Select()
3976 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3979 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3980 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
3983 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
3984 VT == MVT::v2f32) { in Select()
3987 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
3988 VT == MVT::v1f64) { in Select()
3994 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3997 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3998 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4001 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4002 VT == MVT::v2f32) { in Select()
4005 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4006 VT == MVT::v1f64) { in Select()
4015 if (VT == MVT::nxv16i8) { in Select()
4019 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4020 VT == MVT::nxv8bf16) { in Select()
4024 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4028 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4036 if (VT == MVT::nxv16i8) { in Select()
4040 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4041 VT == MVT::nxv8bf16) { in Select()
4045 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4049 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4057 if (VT == MVT::nxv16i8) { in Select()
4061 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4062 VT == MVT::nxv8bf16) { in Select()
4066 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4070 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4080 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, DL, AArch64::FP, MVT::i64); in Select()
4082 CurDAG->getMachineNode(AArch64::SUBXri, DL, MVT::i64, CopyFP, in Select()
4083 CurDAG->getTargetConstant(8, DL, MVT::i32), in Select()
4084 CurDAG->getTargetConstant(0, DL, MVT::i32)), in Select()
4107 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two, in Select()
4111 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three in Select()
4116 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four in Select()
4122 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two, in Select()
4126 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three in Select()
4131 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four in Select()
4151 if (VT == MVT::v8i8) { in Select()
4154 } else if (VT == MVT::v16i8) { in Select()
4157 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4158 VT == MVT::v4bf16) { in Select()
4161 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4162 VT == MVT::v8bf16) { in Select()
4165 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4168 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4171 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4174 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4181 if (VT == MVT::v8i8) { in Select()
4184 } else if (VT == MVT::v16i8) { in Select()
4187 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4188 VT == MVT::v4bf16) { in Select()
4191 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4192 VT == MVT::v8bf16) { in Select()
4195 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4198 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4201 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4204 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4211 if (VT == MVT::v8i8) { in Select()
4214 } else if (VT == MVT::v16i8) { in Select()
4217 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4218 VT == MVT::v4bf16) { in Select()
4221 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4222 VT == MVT::v8bf16) { in Select()
4225 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4228 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4231 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4234 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4241 if (VT == MVT::v8i8) { in Select()
4244 } else if (VT == MVT::v16i8) { in Select()
4247 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4248 VT == MVT::v4bf16) { in Select()
4251 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4252 VT == MVT::v8bf16) { in Select()
4255 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4258 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4261 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4264 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4271 if (VT == MVT::v8i8) { in Select()
4274 } else if (VT == MVT::v16i8) { in Select()
4277 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4278 VT == MVT::v4bf16) { in Select()
4281 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4282 VT == MVT::v8bf16) { in Select()
4285 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4288 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4291 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4294 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4301 if (VT == MVT::v8i8) { in Select()
4304 } else if (VT == MVT::v16i8) { in Select()
4307 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4308 VT == MVT::v4bf16) { in Select()
4311 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4312 VT == MVT::v8bf16) { in Select()
4315 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4318 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4321 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4324 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4331 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4334 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4335 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4338 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4339 VT == MVT::v2f32) { in Select()
4342 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4343 VT == MVT::v1f64) { in Select()
4350 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4353 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4354 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4357 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4358 VT == MVT::v2f32) { in Select()
4361 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4362 VT == MVT::v1f64) { in Select()
4369 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4372 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4373 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4376 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4377 VT == MVT::v2f32) { in Select()
4380 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4381 VT == MVT::v1f64) { in Select()
4388 if (VT == MVT::nxv16i8) { in Select()
4391 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4392 VT == MVT::nxv8bf16) { in Select()
4395 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4398 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4405 if (VT == MVT::nxv16i8) { in Select()
4408 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4409 VT == MVT::nxv8bf16) { in Select()
4412 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4415 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4422 if (VT == MVT::nxv16i8) { in Select()
4425 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4426 VT == MVT::nxv8bf16) { in Select()
4429 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4432 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4442 if (VT == MVT::v8i8) { in Select()
4445 } else if (VT == MVT::v16i8) { in Select()
4448 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4451 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4454 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4457 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4460 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4463 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4470 if (VT == MVT::v8i8) { in Select()
4473 } else if (VT == MVT::v16i8) { in Select()
4476 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4479 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4482 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4485 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4488 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4491 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4498 if (VT == MVT::v8i8) { in Select()
4501 } else if (VT == MVT::v16i8) { in Select()
4504 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4507 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4510 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4513 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4516 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4519 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4526 if (VT == MVT::v8i8) { in Select()
4529 } else if (VT == MVT::v16i8) { in Select()
4532 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4535 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4538 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4541 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4544 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4547 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4554 if (VT == MVT::v8i8) { in Select()
4557 } else if (VT == MVT::v16i8) { in Select()
4560 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4563 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4566 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4569 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4572 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4575 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4582 if (VT == MVT::v8i8) { in Select()
4585 } else if (VT == MVT::v16i8) { in Select()
4588 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4591 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4594 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4597 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4600 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4603 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4610 if (VT == MVT::v8i8) { in Select()
4613 } else if (VT == MVT::v16i8) { in Select()
4616 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4619 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4622 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4625 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4628 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4631 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4638 if (VT == MVT::v8i8) { in Select()
4641 } else if (VT == MVT::v16i8) { in Select()
4644 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4647 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4650 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4653 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4656 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4659 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4666 if (VT == MVT::v8i8) { in Select()
4669 } else if (VT == MVT::v16i8) { in Select()
4672 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4675 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4678 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4681 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4684 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4687 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4694 if (VT == MVT::v8i8) { in Select()
4697 } else if (VT == MVT::v16i8) { in Select()
4700 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4703 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4706 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4709 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4712 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4715 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4722 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4725 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4726 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4729 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4730 VT == MVT::v2f32) { in Select()
4733 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4734 VT == MVT::v1f64) { in Select()
4741 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4744 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4745 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4748 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4749 VT == MVT::v2f32) { in Select()
4752 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4753 VT == MVT::v1f64) { in Select()
4760 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4763 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4764 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4767 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4768 VT == MVT::v2f32) { in Select()
4771 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4772 VT == MVT::v1f64) { in Select()
4779 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4782 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4783 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4786 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4787 VT == MVT::v2f32) { in Select()
4790 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4791 VT == MVT::v1f64) { in Select()
4799 if (VT == MVT::v8i8) { in Select()
4802 } else if (VT == MVT::v16i8) { in Select()
4805 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4808 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4811 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4814 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4817 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4820 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4828 if (VT == MVT::v8i8) { in Select()
4831 } else if (VT == MVT::v16i8) { in Select()
4834 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4837 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4840 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4843 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4846 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4849 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4857 if (VT == MVT::v8i8) { in Select()
4860 } else if (VT == MVT::v16i8) { in Select()
4863 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4866 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4869 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4872 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4875 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4878 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4886 if (VT == MVT::v8i8) { in Select()
4889 } else if (VT == MVT::v16i8) { in Select()
4892 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4895 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4898 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4901 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4904 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4907 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4915 if (VT == MVT::v8i8) { in Select()
4918 } else if (VT == MVT::v16i8) { in Select()
4921 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4924 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) { in Select()
4927 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4930 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4933 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4936 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4944 if (VT == MVT::v8i8) { in Select()
4947 } else if (VT == MVT::v16i8) { in Select()
4950 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4953 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4956 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4959 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4962 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4965 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4973 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4976 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4977 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4980 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4981 VT == MVT::v2f32) { in Select()
4984 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4985 VT == MVT::v1f64) { in Select()
4993 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4996 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4997 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
5000 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
5001 VT == MVT::v2f32) { in Select()
5004 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
5005 VT == MVT::v1f64) { in Select()
5013 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
5016 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
5017 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
5020 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
5021 VT == MVT::v2f32) { in Select()
5024 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
5025 VT == MVT::v1f64) { in Select()
5032 if (VT == MVT::nxv16i8) { in Select()
5035 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5036 VT == MVT::nxv8bf16) { in Select()
5039 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5042 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5049 if (VT == MVT::nxv16i8) { in Select()
5052 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5053 VT == MVT::nxv8bf16) { in Select()
5056 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5059 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5066 if (VT == MVT::nxv16i8) { in Select()
5069 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5070 VT == MVT::nxv8bf16) { in Select()
5073 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5076 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5105 if (!PredVT.isScalableVector() || PredVT.getVectorElementType() != MVT::i1) in getPackedVectorTypeFromPredicateType()
5108 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 && in getPackedVectorTypeFromPredicateType()
5109 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1) in getPackedVectorTypeFromPredicateType()
5160 return MVT::nxv16i8; in getMemVTFromNode()
5188 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectAddrModeIndexedSVE()
5225 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64); in SelectAddrModeIndexedSVE()
5260 Offset = CurDAG->getTargetConstant(ImmOff >> Scale, DL, MVT::i64); in SelectSVERegRegAddrMode()
5262 SDNode *MI = CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops); in SelectSVERegRegAddrMode()
5293 Offset = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectSMETileSlice()
5309 Offset = CurDAG->getTargetConstant(ImmOff, SDLoc(N), MVT::i64); in SelectSMETileSlice()