184   bool isTypeLegal(Type *Ty, MVT &VT);
185   bool isTypeSupported(Type *Ty, MVT &VT, bool IsVectorAllowed = false);
189   bool simplifyAddress(Address &Addr, MVT VT);
198   bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
203   unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
206   unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
209   unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
212   unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
216   unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
224   bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt);
225   bool emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
226   bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS);
227   unsigned emitLoad(MVT VT, MVT ResultVT, Address Addr, bool WantZExt = true,
229   bool emitStore(MVT VT, unsigned SrcReg, Address Addr,
231   bool emitStoreRelease(MVT VT, unsigned SrcReg, unsigned AddrReg,
233   unsigned emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, bool isZExt);
234   unsigned emiti1Ext(unsigned SrcReg, MVT DestVT, bool isZExt);
235   unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
238   unsigned emitAdd_ri_(MVT VT, unsigned Op0, int64_t Imm);
239   unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
242   unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
244   unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
247   unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
249   unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
251   unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
253   unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
254   unsigned emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1);
255   unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
256   unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
257   unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
258   unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
260   unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
261   unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
263   unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
264   unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
267   unsigned materializeInt(const ConstantInt *CI, MVT VT);
268   unsigned materializeFP(const ConstantFP *CFP, MVT VT);
274   bool processCallArgs(CallLoweringInfo &CLI, SmallVectorImpl<MVT> &ArgVTs,
319 static unsigned getImplicitScaleFactor(MVT VT) {  in getImplicitScaleFactor()
323   case MVT::i1:  // fall-through  in getImplicitScaleFactor()
324   case MVT::i8:  in getImplicitScaleFactor()
326   case MVT::i16:  in getImplicitScaleFactor()
328   case MVT::i32: // fall-through  in getImplicitScaleFactor()
329   case MVT::f32:  in getImplicitScaleFactor()
331   case MVT::i64: // fall-through  in getImplicitScaleFactor()
332   case MVT::f64:  in getImplicitScaleFactor()
350   assert(TLI.getValueType(DL, AI->getType(), true) == MVT::i64 &&  in fastMaterializeAlloca()
373 unsigned AArch64FastISel::materializeInt(const ConstantInt *CI, MVT VT) {  in materializeInt()
374   if (VT > MVT::i64)  in materializeInt()
381   const TargetRegisterClass *RC = (VT == MVT::i64) ? &AArch64::GPR64RegClass  in materializeInt()
383   unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;  in materializeInt()
390 unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {  in materializeFP()
396   if (VT != MVT::f32 && VT != MVT::f64)  in materializeFP()
400   bool Is64Bit = (VT == MVT::f64);  in materializeFP()
543   MVT VT = CEVT.getSimpleVT();  in fastMaterializeConstant()
547     assert(VT == MVT::i64 && "Expected 64-bit pointers");  in fastMaterializeConstant()
564   MVT VT;  in fastMaterializeFloatZero()
568   if (VT != MVT::f32 && VT != MVT::f64)  in fastMaterializeFloatZero()
571   bool Is64Bit = (VT == MVT::f64);  in fastMaterializeFloatZero()
784             Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);  in computeAddress()
880         Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);  in computeAddress()
985 bool AArch64FastISel::isTypeLegal(Type *Ty, MVT &VT) {  in isTypeLegal()
992   if (evt == MVT::Other || !evt.isSimple())  in isTypeLegal()
997   if (VT == MVT::f128)  in isTypeLegal()
1009 bool AArch64FastISel::isTypeSupported(Type *Ty, MVT &VT, bool IsVectorAllowed) {  in isTypeSupported()
1018   if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)  in isTypeSupported()
1032 bool AArch64FastISel::simplifyAddress(Address &Addr, MVT VT) {  in simplifyAddress()
1079         ResultReg = emitAddSub_rx(/*UseAdd=*/true, MVT::i64, Addr.getReg(),  in simplifyAddress()
1083         ResultReg = emitAddSub_rs(/*UseAdd=*/true, MVT::i64, Addr.getReg(),  in simplifyAddress()
1088         ResultReg = emitLSL_ri(MVT::i64, MVT::i32, Addr.getOffsetReg(),  in simplifyAddress()
1091         ResultReg = emitLSL_ri(MVT::i64, MVT::i32, Addr.getOffsetReg(),  in simplifyAddress()
1094         ResultReg = emitLSL_ri(MVT::i64, MVT::i64, Addr.getOffsetReg(),  in simplifyAddress()
1112       ResultReg = emitAdd_ri_(MVT::i64, Addr.getReg(), Offset);  in simplifyAddress()
1114       ResultReg = fastEmit_i(MVT::i64, MVT::i64, ISD::Constant, Offset);  in simplifyAddress()
1164 unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,  in emitAddSub()
1172   case MVT::i1:  in emitAddSub()
1175   case MVT::i8:  in emitAddSub()
1179   case MVT::i16:  in emitAddSub()
1183   case MVT::i32:  // fall-through  in emitAddSub()
1184   case MVT::i64:  in emitAddSub()
1187   MVT SrcVT = RetVT;  in emitAddSub()
1188   RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32);  in emitAddSub()
1298 unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rr()
1307   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rr()
1316   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rr()
1335 unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_ri()
1340   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_ri()
1358   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_ri()
1380 unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rs()
1389   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rs()
1402   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rs()
1422 unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rx()
1431   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rx()
1443   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rx()
1471   MVT VT = EVT.getSimpleVT();  in emitCmp()
1476   case MVT::i1:  in emitCmp()
1477   case MVT::i8:  in emitCmp()
1478   case MVT::i16:  in emitCmp()
1479   case MVT::i32:  in emitCmp()
1480   case MVT::i64:  in emitCmp()
1482   case MVT::f32:  in emitCmp()
1483   case MVT::f64:  in emitCmp()
1488 bool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS,  in emitICmp()
1494 bool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm) {  in emitICmp_ri()
1499 bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {  in emitFCmp()
1500   if (RetVT != MVT::f32 && RetVT != MVT::f64)  in emitFCmp()
1515     unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;  in emitFCmp()
1525   unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;  in emitFCmp()
1532 unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,  in emitAdd()
1543 unsigned AArch64FastISel::emitAdd_ri_(MVT VT, unsigned Op0, int64_t Imm) {  in emitAdd_ri_()
1561 unsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,  in emitSub()
1567 unsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg,  in emitSubs_rr()
1573 unsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg,  in emitSubs_rs()
1581 unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp()
1650   MVT VT = std::max(MVT::i32, RetVT.SimpleTy);  in emitLogicalOp()
1652   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp()
1653     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp()
1654     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitLogicalOp()
1659 unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_ri()
1674   case MVT::i1:  in emitLogicalOp_ri()
1675   case MVT::i8:  in emitLogicalOp_ri()
1676   case MVT::i16:  in emitLogicalOp_ri()
1677   case MVT::i32: {  in emitLogicalOp_ri()
1684   case MVT::i64:  in emitLogicalOp_ri()
1697   if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc != ISD::AND) {  in emitLogicalOp_ri()
1698     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_ri()
1699     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitLogicalOp_ri()
1704 unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_rs()
1724   case MVT::i1:  in emitLogicalOp_rs()
1725   case MVT::i8:  in emitLogicalOp_rs()
1726   case MVT::i16:  in emitLogicalOp_rs()
1727   case MVT::i32:  in emitLogicalOp_rs()
1731   case MVT::i64:  in emitLogicalOp_rs()
1739   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp_rs()
1740     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_rs()
1741     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitLogicalOp_rs()
1746 unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg,  in emitAnd_ri()
1751 unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,  in emitLoad()
1827   bool IsRet64Bit = RetVT == MVT::i64;  in emitLoad()
1831   case MVT::i1: // Intentional fall-through.  in emitLoad()
1832   case MVT::i8:  in emitLoad()
1837   case MVT::i16:  in emitLoad()
1842   case MVT::i32:  in emitLoad()
1847   case MVT::i64:  in emitLoad()
1851   case MVT::f32:  in emitLoad()
1855   case MVT::f64:  in emitLoad()
1868   if (VT == MVT::i1) {  in emitLoad()
1869     unsigned ANDReg = emitAnd_ri(MVT::i32, ResultReg, 1);  in emitLoad()
1876   if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {  in emitLoad()
1889   MVT VT;  in selectAddSub()
1915   MVT VT;  in selectLogicalOp()
1944   MVT VT;  in selectLoad()
1974   MVT RetVT = VT;  in selectLoad()
2015       if (RetVT == MVT::i64 && VT <= MVT::i32) {  in selectLoad()
2022           ResultReg = fastEmitInst_extractsubreg(MVT::i32, ResultReg,  in selectLoad()
2054 bool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg,  in emitStoreRelease()
2060   case MVT::i8:  Opc = AArch64::STLRB; break;  in emitStoreRelease()
2061   case MVT::i16: Opc = AArch64::STLRH; break;  in emitStoreRelease()
2062   case MVT::i32: Opc = AArch64::STLRW; break;  in emitStoreRelease()
2063   case MVT::i64: Opc = AArch64::STLRX; break;  in emitStoreRelease()
2076 bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,  in emitStore()
2119   case MVT::i1:  VTIsi1 = true; [[fallthrough]];  in emitStore()
2120   case MVT::i8:  Opc = OpcTable[Idx][0]; break;  in emitStore()
2121   case MVT::i16: Opc = OpcTable[Idx][1]; break;  in emitStore()
2122   case MVT::i32: Opc = OpcTable[Idx][2]; break;  in emitStore()
2123   case MVT::i64: Opc = OpcTable[Idx][3]; break;  in emitStore()
2124   case MVT::f32: Opc = OpcTable[Idx][4]; break;  in emitStore()
2125   case MVT::f64: Opc = OpcTable[Idx][5]; break;  in emitStore()
2130     unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, 1);  in emitStore()
2145   MVT VT;  in selectStore()
2173       SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;  in selectStore()
2176       VT = MVT::getIntegerVT(VT.getSizeInBits());  in selectStore()
2177       SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;  in selectStore()
2273   MVT VT;  in emitCompareAndBranch()
2319     if (VT == MVT::i1)  in emitCompareAndBranch()
2365     SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, AArch64::sub_32);  in emitCompareAndBranch()
2368     SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*isZExt=*/true);  in emitCompareAndBranch()
2550     ResultReg = fastEmit_i(MVT::i32, MVT::i32, ISD::Constant, 1);  in selectCmp()
2660     Src1Reg = emitLogicalOp_ri(ISD::XOR, MVT::i32, Src1Reg, 1);  in optimizeSelect()
2670   MVT VT;  in selectSelect()
2679   case MVT::i1:  in selectSelect()
2680   case MVT::i8:  in selectSelect()
2681   case MVT::i16:  in selectSelect()
2682   case MVT::i32:  in selectSelect()
2686   case MVT::i64:  in selectSelect()
2690   case MVT::f32:  in selectSelect()
2694   case MVT::f64:  in selectSelect()
2822   MVT DestVT;  in selectFPToInt()
2831   if (SrcVT == MVT::f128 || SrcVT == MVT::f16)  in selectFPToInt()
2835   if (SrcVT == MVT::f64) {  in selectFPToInt()
2837       Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;  in selectFPToInt()
2839       Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;  in selectFPToInt()
2842       Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;  in selectFPToInt()
2844       Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;  in selectFPToInt()
2847       DestVT == MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);  in selectFPToInt()
2855   MVT DestVT;  in selectIntToFP()
2859   if (DestVT == MVT::f16)  in selectIntToFP()
2862   assert((DestVT == MVT::f32 || DestVT == MVT::f64) &&  in selectIntToFP()
2872   if (SrcVT == MVT::i16 || SrcVT == MVT::i8 || SrcVT == MVT::i1) {  in selectIntToFP()
2874         emitIntExt(SrcVT.getSimpleVT(), SrcReg, MVT::i32, /*isZExt*/ !Signed);  in selectIntToFP()
2880   if (SrcVT == MVT::i64) {  in selectIntToFP()
2882       Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;  in selectIntToFP()
2884       Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;  in selectIntToFP()
2887       Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;  in selectIntToFP()
2889       Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;  in selectIntToFP()
2933     MVT VT = ArgVT.getSimpleVT().SimpleTy;  in fastLowerArguments()
2941     if (VT >= MVT::i1 && VT <= MVT::i64)  in fastLowerArguments()
2943     else if ((VT >= MVT::f16 && VT <= MVT::f64) || VT.is64BitVector() ||  in fastLowerArguments()
2971     MVT VT = TLI.getSimpleValueType(DL, Arg.getType());  in fastLowerArguments()
2974     if (VT >= MVT::i1 && VT <= MVT::i32) {  in fastLowerArguments()
2977       VT = MVT::i32;  in fastLowerArguments()
2978     } else if (VT == MVT::i64) {  in fastLowerArguments()
2981     } else if (VT == MVT::f16) {  in fastLowerArguments()
2984     } else if (VT ==  MVT::f32) {  in fastLowerArguments()
2987     } else if ((VT == MVT::f64) || VT.is64BitVector()) {  in fastLowerArguments()
3010                                       SmallVectorImpl<MVT> &OutVTs,  in processCallArgs()
3028     MVT ArgVT = OutVTs[VA.getValNo()];  in processCallArgs()
3039       MVT DestVT = VA.getLocVT();  in processCallArgs()
3040       MVT SrcVT = ArgVT;  in processCallArgs()
3049       MVT DestVT = VA.getLocVT();  in processCallArgs()
3050       MVT SrcVT = ArgVT;  in processCallArgs()
3115     MVT CopyVT = VA.getValVT();  in finishCall()
3188   SmallVector<MVT, 16> OutVTs;  in fastLowerCall()
3192     MVT VT;  in fastLowerCall()
3194         !(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16))  in fastLowerCall()
3298     MVT VT;  in tryEmitSmallMemCpy()
3301         VT = MVT::i64;  in tryEmitSmallMemCpy()
3303         VT = MVT::i32;  in tryEmitSmallMemCpy()
3305         VT = MVT::i16;  in tryEmitSmallMemCpy()
3307         VT = MVT::i8;  in tryEmitSmallMemCpy()
3313         VT = MVT::i32;  in tryEmitSmallMemCpy()
3315         VT = MVT::i16;  in tryEmitSmallMemCpy()
3317         VT = MVT::i8;  in tryEmitSmallMemCpy()
3353   MVT RetVT;  in foldXALUIntrinsic()
3360   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in foldXALUIntrinsic()
3533     MVT RetVT;  in fastLowerIntrinsicCall()
3537     if (RetVT != MVT::f32 && RetVT != MVT::f64)  in fastLowerIntrinsicCall()
3546     bool Is64Bit = RetVT == MVT::f64;  in fastLowerIntrinsicCall()
3582     MVT VT;  in fastLowerIntrinsicCall()
3590     case MVT::f32:  in fastLowerIntrinsicCall()
3593     case MVT::f64:  in fastLowerIntrinsicCall()
3618     MVT VT;  in fastLowerIntrinsicCall()
3644     MVT VT;  in fastLowerIntrinsicCall()
3648     if (VT != MVT::i32 && VT != MVT::i64)  in fastLowerIntrinsicCall()
3708       if (VT == MVT::i32) {  in fastLowerIntrinsicCall()
3709         MulReg = emitSMULL_rr(MVT::i64, LHSReg, RHSReg);  in fastLowerIntrinsicCall()
3713         emitAddSub_rx(/*UseAdd=*/false, MVT::i64, MulReg, MulSubReg,  in fastLowerIntrinsicCall()
3718         assert(VT == MVT::i64 && "Unexpected value type.");  in fastLowerIntrinsicCall()
3738       if (VT == MVT::i32) {  in fastLowerIntrinsicCall()
3739         MulReg = emitUMULL_rr(MVT::i64, LHSReg, RHSReg);  in fastLowerIntrinsicCall()
3747         assert(VT == MVT::i64 && "Unexpected value type.");  in fastLowerIntrinsicCall()
3896     MVT RVVT = RVEVT.getSimpleVT();  in selectRet()
3897     if (RVVT == MVT::f128)  in selectRet()
3900     MVT DestVT = VA.getValVT();  in selectRet()
3903       if (RVVT != MVT::i1 && RVVT != MVT::i8 && RVVT != MVT::i16)  in selectRet()
3918       SrcReg = emitAnd_ri(MVT::i64, SrcReg, 0xffffffff);  in selectRet()
3947   MVT SrcVT = SrcEVT.getSimpleVT();  in selectTrunc()
3948   MVT DestVT = DestEVT.getSimpleVT();  in selectTrunc()
3950   if (SrcVT != MVT::i64 && SrcVT != MVT::i32 && SrcVT != MVT::i16 &&  in selectTrunc()
3951       SrcVT != MVT::i8)  in selectTrunc()
3953   if (DestVT != MVT::i32 && DestVT != MVT::i16 && DestVT != MVT::i8 &&  in selectTrunc()
3954       DestVT != MVT::i1)  in selectTrunc()
3967   if (SrcVT == MVT::i64) {  in selectTrunc()
3973     case MVT::i1:  in selectTrunc()
3976     case MVT::i8:  in selectTrunc()
3979     case MVT::i16:  in selectTrunc()
3984     Register Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg,  in selectTrunc()
3987     ResultReg = emitAnd_ri(MVT::i32, Reg32, Mask);  in selectTrunc()
4000 unsigned AArch64FastISel::emiti1Ext(unsigned SrcReg, MVT DestVT, bool IsZExt) {  in emiti1Ext()
4001   assert((DestVT == MVT::i8 || DestVT == MVT::i16 || DestVT == MVT::i32 ||  in emiti1Ext()
4002           DestVT == MVT::i64) &&  in emiti1Ext()
4005   if (DestVT == MVT::i8 || DestVT == MVT::i16)  in emiti1Ext()
4006     DestVT = MVT::i32;  in emiti1Ext()
4009     unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, 1);  in emiti1Ext()
4011     if (DestVT == MVT::i64) {  in emiti1Ext()
4024     if (DestVT == MVT::i64) {  in emiti1Ext()
4033 unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitMul_rr()
4037   case MVT::i8:  in emitMul_rr()
4038   case MVT::i16:  in emitMul_rr()
4039   case MVT::i32:  in emitMul_rr()
4040     RetVT = MVT::i32;  in emitMul_rr()
4042   case MVT::i64:  in emitMul_rr()
4047       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitMul_rr()
4051 unsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitSMULL_rr()
4052   if (RetVT != MVT::i64)  in emitSMULL_rr()
4059 unsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitUMULL_rr()
4060   if (RetVT != MVT::i64)  in emitUMULL_rr()
4067 unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,  in emitLSL_rr()
4074   case MVT::i8:  Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xff;   break;  in emitLSL_rr()
4075   case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xffff; break;  in emitLSL_rr()
4076   case MVT::i32: Opc = AArch64::LSLVWr;                                  break;  in emitLSL_rr()
4077   case MVT::i64: Opc = AArch64::LSLVXr;                                  break;  in emitLSL_rr()
4081       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSL_rr()
4083     Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);  in emitLSL_rr()
4087     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitLSL_rr()
4091 unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSL_ri()
4095   assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||  in emitLSL_ri()
4096           SrcVT == MVT::i32 || SrcVT == MVT::i64) &&  in emitLSL_ri()
4098   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSL_ri()
4099           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSL_ri()
4101   bool Is64Bit = (RetVT == MVT::i64);  in emitLSL_ri()
4157   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSL_ri()
4169 unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,  in emitLSR_rr()
4176   case MVT::i8:  Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xff;   break;  in emitLSR_rr()
4177   case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xffff; break;  in emitLSR_rr()
4178   case MVT::i32: Opc = AArch64::LSRVWr; break;  in emitLSR_rr()
4179   case MVT::i64: Opc = AArch64::LSRVXr; break;  in emitLSR_rr()
4183       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSR_rr()
4185     Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Mask);  in emitLSR_rr()
4186     Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);  in emitLSR_rr()
4190     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitLSR_rr()
4194 unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSR_ri()
4198   assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||  in emitLSR_ri()
4199           SrcVT == MVT::i32 || SrcVT == MVT::i64) &&  in emitLSR_ri()
4201   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSR_ri()
4202           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSR_ri()
4204   bool Is64Bit = (RetVT == MVT::i64);  in emitLSR_ri()
4273   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSR_ri()
4285 unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,  in emitASR_rr()
4292   case MVT::i8:  Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xff;   break;  in emitASR_rr()
4293   case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xffff; break;  in emitASR_rr()
4294   case MVT::i32: Opc = AArch64::ASRVWr;                                  break;  in emitASR_rr()
4295   case MVT::i64: Opc = AArch64::ASRVXr;                                  break;  in emitASR_rr()
4299       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitASR_rr()
4301     Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false);  in emitASR_rr()
4302     Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);  in emitASR_rr()
4306     ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);  in emitASR_rr()
4310 unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitASR_ri()
4314   assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||  in emitASR_ri()
4315           SrcVT == MVT::i32 || SrcVT == MVT::i64) &&  in emitASR_ri()
4317   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitASR_ri()
4318           RetVT == MVT::i64) && "Unexpected return value type.");  in emitASR_ri()
4320   bool Is64Bit = (RetVT == MVT::i64);  in emitASR_ri()
4378   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitASR_ri()
4390 unsigned AArch64FastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,  in emitIntExt()
4392   assert(DestVT != MVT::i1 && "ZeroExt/SignExt an i1?");  in emitIntExt()
4398   if (((DestVT != MVT::i8) && (DestVT != MVT::i16) &&  in emitIntExt()
4399        (DestVT != MVT::i32) && (DestVT != MVT::i64)) ||  in emitIntExt()
4400       ((SrcVT !=  MVT::i1) && (SrcVT !=  MVT::i8) &&  in emitIntExt()
4401        (SrcVT !=  MVT::i16) && (SrcVT !=  MVT::i32)))  in emitIntExt()
4410   case MVT::i1:  in emitIntExt()
4412   case MVT::i8:  in emitIntExt()
4413     if (DestVT == MVT::i64)  in emitIntExt()
4419   case MVT::i16:  in emitIntExt()
4420     if (DestVT == MVT::i64)  in emitIntExt()
4426   case MVT::i32:  in emitIntExt()
4427     assert(DestVT == MVT::i64 && "IntExt i32 to i32?!?");  in emitIntExt()
4434   if (DestVT == MVT::i8 || DestVT == MVT::i16)  in emitIntExt()
4435     DestVT = MVT::i32;  in emitIntExt()
4436   else if (DestVT == MVT::i64) {  in emitIntExt()
4447       (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitIntExt()
4499 bool AArch64FastISel::optimizeIntExtLoad(const Instruction *I, MVT RetVT,  in optimizeIntExtLoad()
4500                                          MVT SrcVT) {  in optimizeIntExtLoad()
4528   if (RetVT != MVT::i64 || SrcVT > MVT::i32) {  in optimizeIntExtLoad()
4556   MVT RetVT;  in selectIntExt()
4557   MVT SrcVT;  in selectIntExt()
4576       if (RetVT == MVT::i64 && SrcVT != MVT::i64) {  in selectIntExt()
4604   MVT DestVT = DestEVT.getSimpleVT();  in selectRem()
4605   if (DestVT != MVT::i64 && DestVT != MVT::i32)  in selectRem()
4609   bool Is64bit = (DestVT == MVT::i64);  in selectRem()
4630       (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in selectRem()
4641   MVT VT;  in selectMul()
4658       MVT SrcVT = VT;  in selectMul()
4662           MVT VT;  in selectMul()
4671           MVT VT;  in selectMul()
4711   MVT RetVT;  in selectShift()
4721     MVT SrcVT = RetVT;  in selectShift()
4726         MVT TmpVT;  in selectShift()
4735         MVT TmpVT;  in selectShift()
4797   MVT RetVT, SrcVT;  in selectBitCast()
4805   if (RetVT == MVT::f32 && SrcVT == MVT::i32)  in selectBitCast()
4807   else if (RetVT == MVT::f64 && SrcVT == MVT::i64)  in selectBitCast()
4809   else if (RetVT == MVT::i32 && SrcVT == MVT::f32)  in selectBitCast()
4811   else if (RetVT == MVT::i64 && SrcVT == MVT::f64)  in selectBitCast()
4819   case MVT::i32: RC = &AArch64::GPR32RegClass; break;  in selectBitCast()
4820   case MVT::i64: RC = &AArch64::GPR64RegClass; break;  in selectBitCast()
4821   case MVT::f32: RC = &AArch64::FPR32RegClass; break;  in selectBitCast()
4822   case MVT::f64: RC = &AArch64::FPR64RegClass; break;  in selectBitCast()
4837   MVT RetVT;  in selectFRem()
4845   case MVT::f32:  in selectFRem()
4848   case MVT::f64:  in selectFRem()
4875   MVT VT;  in selectSDiv()
4883   if ((VT != MVT::i32 && VT != MVT::i64) || !C ||  in selectSDiv()
4911   if (VT == MVT::i64) {  in selectSDiv()
4925   unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;  in selectSDiv()
4950   MVT PtrVT = TLI.getPointerTy(DL);  in getRegForGEPIndex()
4974   MVT VT = TLI.getPointerTy(DL);  in selectGetElementPtr()
5037   MVT VT;  in selectAtomicCmpXchg()
5045   if (VT == MVT::i32) {  in selectAtomicCmpXchg()
5049   } else if (VT == MVT::i64) {  in selectAtomicCmpXchg()
5079       .addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR)  in selectAtomicCmpXchg()