197   bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
202   unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
205   unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
208   unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
211   unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
215   unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
223   bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt);
224   bool emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
225   bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS);
234   unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
238   unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
241   unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
243   unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
246   unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
248   unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
250   unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
252   unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
253   unsigned emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1);
254   unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
255   unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
256   unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
257   unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
259   unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
260   unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
262   unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
263   unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
275   bool finishCall(CallLoweringInfo &CLI, MVT RetVT, unsigned NumBytes);
1137 unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,  in emitAddSub()  argument
1142   switch (RetVT.SimpleTy) {  in emitAddSub()
1160   MVT SrcVT = RetVT;  in emitAddSub()
1161   RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32);  in emitAddSub()
1186     LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);  in emitAddSub()
1192       ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,  in emitAddSub()
1195       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,  in emitAddSub()
1199       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);  in emitAddSub()
1213           return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType,  in emitAddSub()
1219     return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,  in emitAddSub()
1238       ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, AArch64_AM::LSL,  in emitAddSub()
1261           ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,  in emitAddSub()
1275     RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);  in emitAddSub()
1277   return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);  in emitAddSub()
1280 unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rr()  argument
1289   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rr()
1298   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rr()
1317 unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_ri()  argument
1322   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_ri()
1340   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_ri()
1362 unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rs()  argument
1371   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rs()
1375   if (ShiftImm >= RetVT.getSizeInBits())  in emitAddSub_rs()
1384   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rs()
1404 unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rx()  argument
1413   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rx()
1425   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rx()
1470 bool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS,  in emitICmp()  argument
1472   return emitSub(RetVT, LHS, RHS, /*SetFlags=*/true, /*WantResult=*/false,  in emitICmp()
1476 bool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm) {  in emitICmp_ri()  argument
1477   return emitAddSub_ri(/*UseAdd=*/false, RetVT, LHSReg, Imm,  in emitICmp_ri()
1481 bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {  in emitFCmp()  argument
1482   if (RetVT != MVT::f32 && RetVT != MVT::f64)  in emitFCmp()
1497     unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;  in emitFCmp()
1507   unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;  in emitFCmp()
1514 unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,  in emitAdd()  argument
1516   return emitAddSub(/*UseAdd=*/true, RetVT, LHS, RHS, SetFlags, WantResult,  in emitAdd()
1543 unsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,  in emitSub()  argument
1545   return emitAddSub(/*UseAdd=*/false, RetVT, LHS, RHS, SetFlags, WantResult,  in emitSub()
1549 unsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg,  in emitSubs_rr()  argument
1551   return emitAddSub_rr(/*UseAdd=*/false, RetVT, LHSReg, RHSReg,  in emitSubs_rr()
1555 unsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg,  in emitSubs_rs()  argument
1559   return emitAddSub_rs(/*UseAdd=*/false, RetVT, LHSReg, RHSReg, ShiftType,  in emitSubs_rs()
1563 unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp()  argument
1587     ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);  in emitLogicalOp()
1608       ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1622         ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1632   MVT VT = std::max(MVT::i32, RetVT.SimpleTy);  in emitLogicalOp()
1634   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp()
1635     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp()
1641 unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_ri()  argument
1653   switch (RetVT.SimpleTy) {  in emitLogicalOp_ri()
1679   if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc != ISD::AND) {  in emitLogicalOp_ri()
1680     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_ri()
1686 unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_rs()  argument
1698   if (ShiftImm >= RetVT.getSizeInBits())  in emitLogicalOp_rs()
1703   switch (RetVT.SimpleTy) {  in emitLogicalOp_rs()
1721   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp_rs()
1722     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_rs()
1728 unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg,  in emitAnd_ri()  argument
1730   return emitLogicalOp_ri(ISD::AND, RetVT, LHSReg, Imm);  in emitAnd_ri()
1733 unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,  in emitLoad()  argument
1809   bool IsRet64Bit = RetVT == MVT::i64;  in emitLoad()
1858   if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {  in emitLoad()
1956   MVT RetVT = VT;  in selectLoad()  local
1960       if (isTypeSupported(ZE->getType(), RetVT))  in selectLoad()
1963         RetVT = VT;  in selectLoad()
1965       if (isTypeSupported(SE->getType(), RetVT))  in selectLoad()
1968         RetVT = VT;  in selectLoad()
1974       emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));  in selectLoad()
1997       if (RetVT == MVT::i64 && VT <= MVT::i32) {  in selectLoad()
3081 bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,  in finishCall()  argument
3091   if (RetVT != MVT::isVoid) {  in finishCall()
3094     CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));  in finishCall()
3160   MVT RetVT;  in fastLowerCall()  local
3162     RetVT = MVT::isVoid;  in fastLowerCall()
3163   else if (!isTypeLegal(CLI.RetTy, RetVT))  in fastLowerCall()
3261   return finishCall(CLI, RetVT, NumBytes);  in fastLowerCall()
3336   MVT RetVT;  in foldXALUIntrinsic()  local
3340   if (!isTypeLegal(RetTy, RetVT))  in foldXALUIntrinsic()
3343   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in foldXALUIntrinsic()
3514     MVT RetVT;  in fastLowerIntrinsicCall()  local
3515     if (!isTypeLegal(II->getType(), RetVT))  in fastLowerIntrinsicCall()
3518     if (RetVT != MVT::f32 && RetVT != MVT::f64)  in fastLowerIntrinsicCall()
3527     bool Is64Bit = RetVT == MVT::f64;  in fastLowerIntrinsicCall()
3965 unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitMul_rr()  argument
3967   switch (RetVT.SimpleTy) {  in emitMul_rr()
3972     RetVT = MVT::i32;  in emitMul_rr()
3979       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitMul_rr()
3983 unsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitSMULL_rr()  argument
3984   if (RetVT != MVT::i64)  in emitSMULL_rr()
3991 unsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitUMULL_rr()  argument
3992   if (RetVT != MVT::i64)  in emitUMULL_rr()
3999 unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,  in emitLSL_rr()  argument
4004   switch (RetVT.SimpleTy) {  in emitLSL_rr()
4013       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSL_rr()
4023 unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSL_ri()  argument
4025   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSL_ri()
4030   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSL_ri()
4031           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSL_ri()
4033   bool Is64Bit = (RetVT == MVT::i64);  in emitLSL_ri()
4035   unsigned DstBits = RetVT.getSizeInBits();  in emitLSL_ri()
4042     if (RetVT == SrcVT) {  in emitLSL_ri()
4049       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSL_ri()
4089   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSL_ri()
4101 unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,  in emitLSR_rr()  argument
4106   switch (RetVT.SimpleTy) {  in emitLSR_rr()
4115       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSR_rr()
4126 unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSR_ri()  argument
4128   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSR_ri()
4133   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSR_ri()
4134           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSR_ri()
4136   bool Is64Bit = (RetVT == MVT::i64);  in emitLSR_ri()
4138   unsigned DstBits = RetVT.getSizeInBits();  in emitLSR_ri()
4145     if (RetVT == SrcVT) {  in emitLSR_ri()
4152       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4185     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitLSR_ri()
4190     Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4193     SrcVT = RetVT;  in emitLSR_ri()
4205   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSR_ri()
4217 unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,  in emitASR_rr()  argument
4222   switch (RetVT.SimpleTy) {  in emitASR_rr()
4231       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitASR_rr()
4233     Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false);  in emitASR_rr()
4242 unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitASR_ri()  argument
4244   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitASR_ri()
4249   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitASR_ri()
4250           RetVT == MVT::i64) && "Unexpected return value type.");  in emitASR_ri()
4252   bool Is64Bit = (RetVT == MVT::i64);  in emitASR_ri()
4254   unsigned DstBits = RetVT.getSizeInBits();  in emitASR_ri()
4261     if (RetVT == SrcVT) {  in emitASR_ri()
4268       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitASR_ri()
4301     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitASR_ri()
4310   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitASR_ri()
4431 bool AArch64FastISel::optimizeIntExtLoad(const Instruction *I, MVT RetVT,  in optimizeIntExtLoad()  argument
4460   if (RetVT != MVT::i64 || SrcVT > MVT::i32) {  in optimizeIntExtLoad()
4488   MVT RetVT;  in selectIntExt()  local
4490   if (!isTypeSupported(I->getType(), RetVT))  in selectIntExt()
4497   if (optimizeIntExtLoad(I, RetVT, SrcVT))  in selectIntExt()
4508       if (RetVT == MVT::i64 && SrcVT != MVT::i64) {  in selectIntExt()
4523   unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);  in selectIntExt()
4643   MVT RetVT;  in selectShift()  local
4644   if (!isTypeSupported(I->getType(), RetVT, /*IsVectorAllowed=*/true))  in selectShift()
4647   if (RetVT.isVector())  in selectShift()
4653     MVT SrcVT = RetVT;  in selectShift()
4683       ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4686       ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4689       ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4711     ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4714     ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4717     ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4729   MVT RetVT, SrcVT;  in selectBitCast()  local
4733   if (!isTypeLegal(I->getType(), RetVT))  in selectBitCast()
4737   if (RetVT == MVT::f32 && SrcVT == MVT::i32)  in selectBitCast()
4739   else if (RetVT == MVT::f64 && SrcVT == MVT::i64)  in selectBitCast()
4741   else if (RetVT == MVT::i32 && SrcVT == MVT::f32)  in selectBitCast()
4743   else if (RetVT == MVT::i64 && SrcVT == MVT::f64)  in selectBitCast()
4749   switch (RetVT.SimpleTy) {  in selectBitCast()
4769   MVT RetVT;  in selectFRem()  local
4770   if (!isTypeLegal(I->getType(), RetVT))  in selectFRem()
4774   switch (RetVT.SimpleTy) {  in selectFRem()