198   bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
203   unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
206   unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
209   unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
212   unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
216   unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
224   bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt);
225   bool emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
226   bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS);
235   unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
239   unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
242   unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
244   unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
247   unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
249   unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
251   unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
253   unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
254   unsigned emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1);
255   unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
256   unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
257   unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
258   unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
260   unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
261   unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
263   unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
264   unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
1164 unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,  in emitAddSub()  argument
1169   switch (RetVT.SimpleTy) {  in emitAddSub()
1187   MVT SrcVT = RetVT;  in emitAddSub()
1188   RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32);  in emitAddSub()
1213     LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);  in emitAddSub()
1219       ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,  in emitAddSub()
1222       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,  in emitAddSub()
1226       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);  in emitAddSub()
1237     return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,  in emitAddSub()
1256       ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, AArch64_AM::LSL,  in emitAddSub()
1279           ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,  in emitAddSub()
1293     RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);  in emitAddSub()
1295   return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);  in emitAddSub()
1298 unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rr()  argument
1307   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rr()
1316   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rr()
1335 unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_ri()  argument
1340   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_ri()
1358   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_ri()
1380 unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rs()  argument
1389   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rs()
1393   if (ShiftImm >= RetVT.getSizeInBits())  in emitAddSub_rs()
1402   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rs()
1422 unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rx()  argument
1431   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rx()
1443   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rx()
1488 bool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS,  in emitICmp()  argument
1490   return emitSub(RetVT, LHS, RHS, /*SetFlags=*/true, /*WantResult=*/false,  in emitICmp()
1494 bool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm) {  in emitICmp_ri()  argument
1495   return emitAddSub_ri(/*UseAdd=*/false, RetVT, LHSReg, Imm,  in emitICmp_ri()
1499 bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {  in emitFCmp()  argument
1500   if (RetVT != MVT::f32 && RetVT != MVT::f64)  in emitFCmp()
1515     unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;  in emitFCmp()
1525   unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;  in emitFCmp()
1532 unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,  in emitAdd()  argument
1534   return emitAddSub(/*UseAdd=*/true, RetVT, LHS, RHS, SetFlags, WantResult,  in emitAdd()
1561 unsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,  in emitSub()  argument
1563   return emitAddSub(/*UseAdd=*/false, RetVT, LHS, RHS, SetFlags, WantResult,  in emitSub()
1567 unsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg,  in emitSubs_rr()  argument
1569   return emitAddSub_rr(/*UseAdd=*/false, RetVT, LHSReg, RHSReg,  in emitSubs_rr()
1573 unsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg,  in emitSubs_rs()  argument
1577   return emitAddSub_rs(/*UseAdd=*/false, RetVT, LHSReg, RHSReg, ShiftType,  in emitSubs_rs()
1581 unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp()  argument
1605     ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);  in emitLogicalOp()
1626       ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1640         ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1650   MVT VT = std::max(MVT::i32, RetVT.SimpleTy);  in emitLogicalOp()
1652   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp()
1653     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp()
1659 unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_ri()  argument
1671   switch (RetVT.SimpleTy) {  in emitLogicalOp_ri()
1697   if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc != ISD::AND) {  in emitLogicalOp_ri()
1698     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_ri()
1704 unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_rs()  argument
1716   if (ShiftImm >= RetVT.getSizeInBits())  in emitLogicalOp_rs()
1721   switch (RetVT.SimpleTy) {  in emitLogicalOp_rs()
1739   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp_rs()
1740     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_rs()
1746 unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg,  in emitAnd_ri()  argument
1748   return emitLogicalOp_ri(ISD::AND, RetVT, LHSReg, Imm);  in emitAnd_ri()
1751 unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,  in emitLoad()  argument
1827   bool IsRet64Bit = RetVT == MVT::i64;  in emitLoad()
1876   if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {  in emitLoad()
1974   MVT RetVT = VT;  in selectLoad()  local
1978       if (isTypeSupported(ZE->getType(), RetVT))  in selectLoad()
1981         RetVT = VT;  in selectLoad()
1983       if (isTypeSupported(SE->getType(), RetVT))  in selectLoad()
1986         RetVT = VT;  in selectLoad()
1992       emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));  in selectLoad()
2015       if (RetVT == MVT::i64 && VT <= MVT::i32) {  in selectLoad()
3353   MVT RetVT;  in foldXALUIntrinsic()  local
3357   if (!isTypeLegal(RetTy, RetVT))  in foldXALUIntrinsic()
3360   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in foldXALUIntrinsic()
3533     MVT RetVT;  in fastLowerIntrinsicCall()  local
3534     if (!isTypeLegal(II->getType(), RetVT))  in fastLowerIntrinsicCall()
3537     if (RetVT != MVT::f32 && RetVT != MVT::f64)  in fastLowerIntrinsicCall()
3546     bool Is64Bit = RetVT == MVT::f64;  in fastLowerIntrinsicCall()
4033 unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitMul_rr()  argument
4035   switch (RetVT.SimpleTy) {  in emitMul_rr()
4040     RetVT = MVT::i32;  in emitMul_rr()
4047       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitMul_rr()
4051 unsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitSMULL_rr()  argument
4052   if (RetVT != MVT::i64)  in emitSMULL_rr()
4059 unsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitUMULL_rr()  argument
4060   if (RetVT != MVT::i64)  in emitUMULL_rr()
4067 unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,  in emitLSL_rr()  argument
4072   switch (RetVT.SimpleTy) {  in emitLSL_rr()
4081       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSL_rr()
4091 unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSL_ri()  argument
4093   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSL_ri()
4098   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSL_ri()
4099           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSL_ri()
4101   bool Is64Bit = (RetVT == MVT::i64);  in emitLSL_ri()
4103   unsigned DstBits = RetVT.getSizeInBits();  in emitLSL_ri()
4110     if (RetVT == SrcVT) {  in emitLSL_ri()
4117       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSL_ri()
4157   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSL_ri()
4169 unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,  in emitLSR_rr()  argument
4174   switch (RetVT.SimpleTy) {  in emitLSR_rr()
4183       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSR_rr()
4194 unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSR_ri()  argument
4196   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSR_ri()
4201   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSR_ri()
4202           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSR_ri()
4204   bool Is64Bit = (RetVT == MVT::i64);  in emitLSR_ri()
4206   unsigned DstBits = RetVT.getSizeInBits();  in emitLSR_ri()
4213     if (RetVT == SrcVT) {  in emitLSR_ri()
4220       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4253     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitLSR_ri()
4258     Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4261     SrcVT = RetVT;  in emitLSR_ri()
4273   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSR_ri()
4285 unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,  in emitASR_rr()  argument
4290   switch (RetVT.SimpleTy) {  in emitASR_rr()
4299       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitASR_rr()
4301     Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false);  in emitASR_rr()
4310 unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitASR_ri()  argument
4312   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitASR_ri()
4317   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitASR_ri()
4318           RetVT == MVT::i64) && "Unexpected return value type.");  in emitASR_ri()
4320   bool Is64Bit = (RetVT == MVT::i64);  in emitASR_ri()
4322   unsigned DstBits = RetVT.getSizeInBits();  in emitASR_ri()
4329     if (RetVT == SrcVT) {  in emitASR_ri()
4336       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitASR_ri()
4369     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitASR_ri()
4378   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitASR_ri()
4499 bool AArch64FastISel::optimizeIntExtLoad(const Instruction *I, MVT RetVT,  in optimizeIntExtLoad()  argument
4528   if (RetVT != MVT::i64 || SrcVT > MVT::i32) {  in optimizeIntExtLoad()
4556   MVT RetVT;  in selectIntExt()  local
4558   if (!isTypeSupported(I->getType(), RetVT))  in selectIntExt()
4565   if (optimizeIntExtLoad(I, RetVT, SrcVT))  in selectIntExt()
4576       if (RetVT == MVT::i64 && SrcVT != MVT::i64) {  in selectIntExt()
4591   unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);  in selectIntExt()
4711   MVT RetVT;  in selectShift()  local
4712   if (!isTypeSupported(I->getType(), RetVT, /*IsVectorAllowed=*/true))  in selectShift()
4715   if (RetVT.isVector())  in selectShift()
4721     MVT SrcVT = RetVT;  in selectShift()
4751       ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4754       ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4757       ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4779     ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4782     ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4785     ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4797   MVT RetVT, SrcVT;  in selectBitCast()  local
4801   if (!isTypeLegal(I->getType(), RetVT))  in selectBitCast()
4805   if (RetVT == MVT::f32 && SrcVT == MVT::i32)  in selectBitCast()
4807   else if (RetVT == MVT::f64 && SrcVT == MVT::i64)  in selectBitCast()
4809   else if (RetVT == MVT::i32 && SrcVT == MVT::f32)  in selectBitCast()
4811   else if (RetVT == MVT::i64 && SrcVT == MVT::f64)  in selectBitCast()
4817   switch (RetVT.SimpleTy) {  in selectBitCast()
4837   MVT RetVT;  in selectFRem()  local
4838   if (!isTypeLegal(I->getType(), RetVT))  in selectFRem()
4842   switch (RetVT.SimpleTy) {  in selectFRem()