1 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the Base ARM implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARMBaseInstrInfo.h"
14 #include "ARMBaseRegisterInfo.h"
15 #include "ARMConstantPoolValue.h"
16 #include "ARMFeatures.h"
17 #include "ARMHazardRecognizer.h"
18 #include "ARMMachineFunctionInfo.h"
19 #include "ARMSubtarget.h"
20 #include "MCTargetDesc/ARMAddressingModes.h"
21 #include "MCTargetDesc/ARMBaseInfo.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/Triple.h"
27 #include "llvm/CodeGen/LiveVariables.h"
28 #include "llvm/CodeGen/MachineBasicBlock.h"
29 #include "llvm/CodeGen/MachineConstantPool.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstr.h"
33 #include "llvm/CodeGen/MachineInstrBuilder.h"
34 #include "llvm/CodeGen/MachineMemOperand.h"
35 #include "llvm/CodeGen/MachineModuleInfo.h"
36 #include "llvm/CodeGen/MachineOperand.h"
37 #include "llvm/CodeGen/MachineRegisterInfo.h"
38 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
39 #include "llvm/CodeGen/SelectionDAGNodes.h"
40 #include "llvm/CodeGen/TargetInstrInfo.h"
41 #include "llvm/CodeGen/TargetRegisterInfo.h"
42 #include "llvm/CodeGen/TargetSchedule.h"
43 #include "llvm/IR/Attributes.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DebugLoc.h"
46 #include "llvm/IR/Function.h"
47 #include "llvm/IR/GlobalValue.h"
48 #include "llvm/MC/MCAsmInfo.h"
49 #include "llvm/MC/MCInstrDesc.h"
50 #include "llvm/MC/MCInstrItineraries.h"
51 #include "llvm/Support/BranchProbability.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/CommandLine.h"
54 #include "llvm/Support/Compiler.h"
55 #include "llvm/Support/Debug.h"
56 #include "llvm/Support/ErrorHandling.h"
57 #include "llvm/Support/raw_ostream.h"
58 #include "llvm/Target/TargetMachine.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <iterator>
63 #include <new>
64 #include <utility>
65 #include <vector>
66 
67 using namespace llvm;
68 
69 #define DEBUG_TYPE "arm-instrinfo"
70 
71 #define GET_INSTRINFO_CTOR_DTOR
72 #include "ARMGenInstrInfo.inc"
73 
74 static cl::opt<bool>
75 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
76                cl::desc("Enable ARM 2-addr to 3-addr conv"));
77 
78 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
79 struct ARM_MLxEntry {
80   uint16_t MLxOpc;     // MLA / MLS opcode
81   uint16_t MulOpc;     // Expanded multiplication opcode
82   uint16_t AddSubOpc;  // Expanded add / sub opcode
83   bool NegAcc;         // True if the acc is negated before the add / sub.
84   bool HasLane;        // True if instruction has an extra "lane" operand.
85 };
86 
87 static const ARM_MLxEntry ARM_MLxTable[] = {
88   // MLxOpc,          MulOpc,           AddSubOpc,       NegAcc, HasLane
89   // fp scalar ops
90   { ARM::VMLAS,       ARM::VMULS,       ARM::VADDS,      false,  false },
91   { ARM::VMLSS,       ARM::VMULS,       ARM::VSUBS,      false,  false },
92   { ARM::VMLAD,       ARM::VMULD,       ARM::VADDD,      false,  false },
93   { ARM::VMLSD,       ARM::VMULD,       ARM::VSUBD,      false,  false },
94   { ARM::VNMLAS,      ARM::VNMULS,      ARM::VSUBS,      true,   false },
95   { ARM::VNMLSS,      ARM::VMULS,       ARM::VSUBS,      true,   false },
96   { ARM::VNMLAD,      ARM::VNMULD,      ARM::VSUBD,      true,   false },
97   { ARM::VNMLSD,      ARM::VMULD,       ARM::VSUBD,      true,   false },
98 
99   // fp SIMD ops
100   { ARM::VMLAfd,      ARM::VMULfd,      ARM::VADDfd,     false,  false },
101   { ARM::VMLSfd,      ARM::VMULfd,      ARM::VSUBfd,     false,  false },
102   { ARM::VMLAfq,      ARM::VMULfq,      ARM::VADDfq,     false,  false },
103   { ARM::VMLSfq,      ARM::VMULfq,      ARM::VSUBfq,     false,  false },
104   { ARM::VMLAslfd,    ARM::VMULslfd,    ARM::VADDfd,     false,  true  },
105   { ARM::VMLSslfd,    ARM::VMULslfd,    ARM::VSUBfd,     false,  true  },
106   { ARM::VMLAslfq,    ARM::VMULslfq,    ARM::VADDfq,     false,  true  },
107   { ARM::VMLSslfq,    ARM::VMULslfq,    ARM::VSUBfq,     false,  true  },
108 };
109 
110 ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI)
111   : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
112     Subtarget(STI) {
113   for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) {
114     if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second)
115       llvm_unreachable("Duplicated entries?");
116     MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc);
117     MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc);
118   }
119 }
120 
121 // Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
122 // currently defaults to no prepass hazard recognizer.
123 ScheduleHazardRecognizer *
124 ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
125                                                const ScheduleDAG *DAG) const {
126   if (usePreRAHazardRecognizer()) {
127     const InstrItineraryData *II =
128         static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
129     return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
130   }
131   return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
132 }
133 
134 ScheduleHazardRecognizer *ARMBaseInstrInfo::
135 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
136                                    const ScheduleDAG *DAG) const {
137   if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
138     return new ARMHazardRecognizer(II, DAG);
139   return TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
140 }
141 
142 MachineInstr *ARMBaseInstrInfo::convertToThreeAddress(
143     MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const {
144   // FIXME: Thumb2 support.
145 
146   if (!EnableARM3Addr)
147     return nullptr;
148 
149   MachineFunction &MF = *MI.getParent()->getParent();
150   uint64_t TSFlags = MI.getDesc().TSFlags;
151   bool isPre = false;
152   switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) {
153   default: return nullptr;
154   case ARMII::IndexModePre:
155     isPre = true;
156     break;
157   case ARMII::IndexModePost:
158     break;
159   }
160 
161   // Try splitting an indexed load/store to an un-indexed one plus an add/sub
162   // operation.
163   unsigned MemOpc = getUnindexedOpcode(MI.getOpcode());
164   if (MemOpc == 0)
165     return nullptr;
166 
167   MachineInstr *UpdateMI = nullptr;
168   MachineInstr *MemMI = nullptr;
169   unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
170   const MCInstrDesc &MCID = MI.getDesc();
171   unsigned NumOps = MCID.getNumOperands();
172   bool isLoad = !MI.mayStore();
173   const MachineOperand &WB = isLoad ? MI.getOperand(1) : MI.getOperand(0);
174   const MachineOperand &Base = MI.getOperand(2);
175   const MachineOperand &Offset = MI.getOperand(NumOps - 3);
176   Register WBReg = WB.getReg();
177   Register BaseReg = Base.getReg();
178   Register OffReg = Offset.getReg();
179   unsigned OffImm = MI.getOperand(NumOps - 2).getImm();
180   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI.getOperand(NumOps - 1).getImm();
181   switch (AddrMode) {
182   default: llvm_unreachable("Unknown indexed op!");
183   case ARMII::AddrMode2: {
184     bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
185     unsigned Amt = ARM_AM::getAM2Offset(OffImm);
186     if (OffReg == 0) {
187       if (ARM_AM::getSOImmVal(Amt) == -1)
188         // Can't encode it in a so_imm operand. This transformation will
189         // add more than 1 instruction. Abandon!
190         return nullptr;
191       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
192                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
193                      .addReg(BaseReg)
194                      .addImm(Amt)
195                      .add(predOps(Pred))
196                      .add(condCodeOp());
197     } else if (Amt != 0) {
198       ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
199       unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
200       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
201                          get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
202                      .addReg(BaseReg)
203                      .addReg(OffReg)
204                      .addReg(0)
205                      .addImm(SOOpc)
206                      .add(predOps(Pred))
207                      .add(condCodeOp());
208     } else
209       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
210                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
211                      .addReg(BaseReg)
212                      .addReg(OffReg)
213                      .add(predOps(Pred))
214                      .add(condCodeOp());
215     break;
216   }
217   case ARMII::AddrMode3 : {
218     bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub;
219     unsigned Amt = ARM_AM::getAM3Offset(OffImm);
220     if (OffReg == 0)
221       // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand.
222       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
223                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
224                      .addReg(BaseReg)
225                      .addImm(Amt)
226                      .add(predOps(Pred))
227                      .add(condCodeOp());
228     else
229       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
230                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
231                      .addReg(BaseReg)
232                      .addReg(OffReg)
233                      .add(predOps(Pred))
234                      .add(condCodeOp());
235     break;
236   }
237   }
238 
239   std::vector<MachineInstr*> NewMIs;
240   if (isPre) {
241     if (isLoad)
242       MemMI =
243           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
244               .addReg(WBReg)
245               .addImm(0)
246               .addImm(Pred);
247     else
248       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
249                   .addReg(MI.getOperand(1).getReg())
250                   .addReg(WBReg)
251                   .addReg(0)
252                   .addImm(0)
253                   .addImm(Pred);
254     NewMIs.push_back(MemMI);
255     NewMIs.push_back(UpdateMI);
256   } else {
257     if (isLoad)
258       MemMI =
259           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
260               .addReg(BaseReg)
261               .addImm(0)
262               .addImm(Pred);
263     else
264       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
265                   .addReg(MI.getOperand(1).getReg())
266                   .addReg(BaseReg)
267                   .addReg(0)
268                   .addImm(0)
269                   .addImm(Pred);
270     if (WB.isDead())
271       UpdateMI->getOperand(0).setIsDead();
272     NewMIs.push_back(UpdateMI);
273     NewMIs.push_back(MemMI);
274   }
275 
276   // Transfer LiveVariables states, kill / dead info.
277   if (LV) {
278     for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
279       MachineOperand &MO = MI.getOperand(i);
280       if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) {
281         Register Reg = MO.getReg();
282 
283         LiveVariables::VarInfo &VI = LV->getVarInfo(Reg);
284         if (MO.isDef()) {
285           MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI;
286           if (MO.isDead())
287             LV->addVirtualRegisterDead(Reg, *NewMI);
288         }
289         if (MO.isUse() && MO.isKill()) {
290           for (unsigned j = 0; j < 2; ++j) {
291             // Look at the two new MI's in reverse order.
292             MachineInstr *NewMI = NewMIs[j];
293             if (!NewMI->readsRegister(Reg))
294               continue;
295             LV->addVirtualRegisterKilled(Reg, *NewMI);
296             if (VI.removeKill(MI))
297               VI.Kills.push_back(NewMI);
298             break;
299           }
300         }
301       }
302     }
303   }
304 
305   MachineBasicBlock::iterator MBBI = MI.getIterator();
306   MFI->insert(MBBI, NewMIs[1]);
307   MFI->insert(MBBI, NewMIs[0]);
308   return NewMIs[0];
309 }
310 
311 // Branch analysis.
312 bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
313                                      MachineBasicBlock *&TBB,
314                                      MachineBasicBlock *&FBB,
315                                      SmallVectorImpl<MachineOperand> &Cond,
316                                      bool AllowModify) const {
317   TBB = nullptr;
318   FBB = nullptr;
319 
320   MachineBasicBlock::iterator I = MBB.end();
321   if (I == MBB.begin())
322     return false; // Empty blocks are easy.
323   --I;
324 
325   // Walk backwards from the end of the basic block until the branch is
326   // analyzed or we give up.
327   while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) {
328     // Flag to be raised on unanalyzeable instructions. This is useful in cases
329     // where we want to clean up on the end of the basic block before we bail
330     // out.
331     bool CantAnalyze = false;
332 
333     // Skip over DEBUG values and predicated nonterminators.
334     while (I->isDebugInstr() || !I->isTerminator()) {
335       if (I == MBB.begin())
336         return false;
337       --I;
338     }
339 
340     if (isIndirectBranchOpcode(I->getOpcode()) ||
341         isJumpTableBranchOpcode(I->getOpcode())) {
342       // Indirect branches and jump tables can't be analyzed, but we still want
343       // to clean up any instructions at the tail of the basic block.
344       CantAnalyze = true;
345     } else if (isUncondBranchOpcode(I->getOpcode())) {
346       TBB = I->getOperand(0).getMBB();
347     } else if (isCondBranchOpcode(I->getOpcode())) {
348       // Bail out if we encounter multiple conditional branches.
349       if (!Cond.empty())
350         return true;
351 
352       assert(!FBB && "FBB should have been null.");
353       FBB = TBB;
354       TBB = I->getOperand(0).getMBB();
355       Cond.push_back(I->getOperand(1));
356       Cond.push_back(I->getOperand(2));
357     } else if (I->isReturn()) {
358       // Returns can't be analyzed, but we should run cleanup.
359       CantAnalyze = !isPredicated(*I);
360     } else {
361       // We encountered other unrecognized terminator. Bail out immediately.
362       return true;
363     }
364 
365     // Cleanup code - to be run for unpredicated unconditional branches and
366     //                returns.
367     if (!isPredicated(*I) &&
368           (isUncondBranchOpcode(I->getOpcode()) ||
369            isIndirectBranchOpcode(I->getOpcode()) ||
370            isJumpTableBranchOpcode(I->getOpcode()) ||
371            I->isReturn())) {
372       // Forget any previous condition branch information - it no longer applies.
373       Cond.clear();
374       FBB = nullptr;
375 
376       // If we can modify the function, delete everything below this
377       // unconditional branch.
378       if (AllowModify) {
379         MachineBasicBlock::iterator DI = std::next(I);
380         while (DI != MBB.end()) {
381           MachineInstr &InstToDelete = *DI;
382           ++DI;
383           InstToDelete.eraseFromParent();
384         }
385       }
386     }
387 
388     if (CantAnalyze)
389       return true;
390 
391     if (I == MBB.begin())
392       return false;
393 
394     --I;
395   }
396 
397   // We made it past the terminators without bailing out - we must have
398   // analyzed this branch successfully.
399   return false;
400 }
401 
402 unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
403                                         int *BytesRemoved) const {
404   assert(!BytesRemoved && "code size not handled");
405 
406   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
407   if (I == MBB.end())
408     return 0;
409 
410   if (!isUncondBranchOpcode(I->getOpcode()) &&
411       !isCondBranchOpcode(I->getOpcode()))
412     return 0;
413 
414   // Remove the branch.
415   I->eraseFromParent();
416 
417   I = MBB.end();
418 
419   if (I == MBB.begin()) return 1;
420   --I;
421   if (!isCondBranchOpcode(I->getOpcode()))
422     return 1;
423 
424   // Remove the branch.
425   I->eraseFromParent();
426   return 2;
427 }
428 
429 unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
430                                         MachineBasicBlock *TBB,
431                                         MachineBasicBlock *FBB,
432                                         ArrayRef<MachineOperand> Cond,
433                                         const DebugLoc &DL,
434                                         int *BytesAdded) const {
435   assert(!BytesAdded && "code size not handled");
436   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
437   int BOpc   = !AFI->isThumbFunction()
438     ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
439   int BccOpc = !AFI->isThumbFunction()
440     ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
441   bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
442 
443   // Shouldn't be a fall through.
444   assert(TBB && "insertBranch must not be told to insert a fallthrough");
445   assert((Cond.size() == 2 || Cond.size() == 0) &&
446          "ARM branch conditions have two components!");
447 
448   // For conditional branches, we use addOperand to preserve CPSR flags.
449 
450   if (!FBB) {
451     if (Cond.empty()) { // Unconditional branch?
452       if (isThumb)
453         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).add(predOps(ARMCC::AL));
454       else
455         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
456     } else
457       BuildMI(&MBB, DL, get(BccOpc))
458           .addMBB(TBB)
459           .addImm(Cond[0].getImm())
460           .add(Cond[1]);
461     return 1;
462   }
463 
464   // Two-way conditional branch.
465   BuildMI(&MBB, DL, get(BccOpc))
466       .addMBB(TBB)
467       .addImm(Cond[0].getImm())
468       .add(Cond[1]);
469   if (isThumb)
470     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).add(predOps(ARMCC::AL));
471   else
472     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
473   return 2;
474 }
475 
476 bool ARMBaseInstrInfo::
477 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
478   ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
479   Cond[0].setImm(ARMCC::getOppositeCondition(CC));
480   return false;
481 }
482 
483 bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
484   if (MI.isBundle()) {
485     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
486     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
487     while (++I != E && I->isInsideBundle()) {
488       int PIdx = I->findFirstPredOperandIdx();
489       if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL)
490         return true;
491     }
492     return false;
493   }
494 
495   int PIdx = MI.findFirstPredOperandIdx();
496   return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL;
497 }
498 
499 std::string ARMBaseInstrInfo::createMIROperandComment(
500     const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
501     const TargetRegisterInfo *TRI) const {
502 
503   // First, let's see if there is a generic comment for this operand
504   std::string GenericComment =
505       TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
506   if (!GenericComment.empty())
507     return GenericComment;
508 
509   // If not, check if we have an immediate operand.
510   if (Op.getType() != MachineOperand::MO_Immediate)
511     return std::string();
512 
513   // And print its corresponding condition code if the immediate is a
514   // predicate.
515   int FirstPredOp = MI.findFirstPredOperandIdx();
516   if (FirstPredOp != (int) OpIdx)
517     return std::string();
518 
519   std::string CC = "CC::";
520   CC += ARMCondCodeToString((ARMCC::CondCodes)Op.getImm());
521   return CC;
522 }
523 
524 bool ARMBaseInstrInfo::PredicateInstruction(
525     MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
526   unsigned Opc = MI.getOpcode();
527   if (isUncondBranchOpcode(Opc)) {
528     MI.setDesc(get(getMatchingCondBranchOpcode(Opc)));
529     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
530       .addImm(Pred[0].getImm())
531       .addReg(Pred[1].getReg());
532     return true;
533   }
534 
535   int PIdx = MI.findFirstPredOperandIdx();
536   if (PIdx != -1) {
537     MachineOperand &PMO = MI.getOperand(PIdx);
538     PMO.setImm(Pred[0].getImm());
539     MI.getOperand(PIdx+1).setReg(Pred[1].getReg());
540     return true;
541   }
542   return false;
543 }
544 
545 bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
546                                          ArrayRef<MachineOperand> Pred2) const {
547   if (Pred1.size() > 2 || Pred2.size() > 2)
548     return false;
549 
550   ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
551   ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
552   if (CC1 == CC2)
553     return true;
554 
555   switch (CC1) {
556   default:
557     return false;
558   case ARMCC::AL:
559     return true;
560   case ARMCC::HS:
561     return CC2 == ARMCC::HI;
562   case ARMCC::LS:
563     return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
564   case ARMCC::GE:
565     return CC2 == ARMCC::GT;
566   case ARMCC::LE:
567     return CC2 == ARMCC::LT;
568   }
569 }
570 
571 bool ARMBaseInstrInfo::DefinesPredicate(
572     MachineInstr &MI, std::vector<MachineOperand> &Pred) const {
573   bool Found = false;
574   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
575     const MachineOperand &MO = MI.getOperand(i);
576     if ((MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) ||
577         (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)) {
578       Pred.push_back(MO);
579       Found = true;
580     }
581   }
582 
583   return Found;
584 }
585 
586 bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
587   for (const auto &MO : MI.operands())
588     if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
589       return true;
590   return false;
591 }
592 
593 bool ARMBaseInstrInfo::isAddrMode3OpImm(const MachineInstr &MI,
594                                         unsigned Op) const {
595   const MachineOperand &Offset = MI.getOperand(Op + 1);
596   return Offset.getReg() != 0;
597 }
598 
599 // Load with negative register offset requires additional 1cyc and +I unit
600 // for Cortex A57
601 bool ARMBaseInstrInfo::isAddrMode3OpMinusReg(const MachineInstr &MI,
602                                              unsigned Op) const {
603   const MachineOperand &Offset = MI.getOperand(Op + 1);
604   const MachineOperand &Opc = MI.getOperand(Op + 2);
605   assert(Opc.isImm());
606   assert(Offset.isReg());
607   int64_t OpcImm = Opc.getImm();
608 
609   bool isSub = ARM_AM::getAM3Op(OpcImm) == ARM_AM::sub;
610   return (isSub && Offset.getReg() != 0);
611 }
612 
613 bool ARMBaseInstrInfo::isLdstScaledReg(const MachineInstr &MI,
614                                        unsigned Op) const {
615   const MachineOperand &Opc = MI.getOperand(Op + 2);
616   unsigned OffImm = Opc.getImm();
617   return ARM_AM::getAM2ShiftOpc(OffImm) != ARM_AM::no_shift;
618 }
619 
620 // Load, scaled register offset, not plus LSL2
621 bool ARMBaseInstrInfo::isLdstScaledRegNotPlusLsl2(const MachineInstr &MI,
622                                                   unsigned Op) const {
623   const MachineOperand &Opc = MI.getOperand(Op + 2);
624   unsigned OffImm = Opc.getImm();
625 
626   bool isAdd = ARM_AM::getAM2Op(OffImm) == ARM_AM::add;
627   unsigned Amt = ARM_AM::getAM2Offset(OffImm);
628   ARM_AM::ShiftOpc ShiftOpc = ARM_AM::getAM2ShiftOpc(OffImm);
629   if (ShiftOpc == ARM_AM::no_shift) return false; // not scaled
630   bool SimpleScaled = (isAdd && ShiftOpc == ARM_AM::lsl && Amt == 2);
631   return !SimpleScaled;
632 }
633 
634 // Minus reg for ldstso addr mode
635 bool ARMBaseInstrInfo::isLdstSoMinusReg(const MachineInstr &MI,
636                                         unsigned Op) const {
637   unsigned OffImm = MI.getOperand(Op + 2).getImm();
638   return ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
639 }
640 
641 // Load, scaled register offset
642 bool ARMBaseInstrInfo::isAm2ScaledReg(const MachineInstr &MI,
643                                       unsigned Op) const {
644   unsigned OffImm = MI.getOperand(Op + 2).getImm();
645   return ARM_AM::getAM2ShiftOpc(OffImm) != ARM_AM::no_shift;
646 }
647 
648 static bool isEligibleForITBlock(const MachineInstr *MI) {
649   switch (MI->getOpcode()) {
650   default: return true;
651   case ARM::tADC:   // ADC (register) T1
652   case ARM::tADDi3: // ADD (immediate) T1
653   case ARM::tADDi8: // ADD (immediate) T2
654   case ARM::tADDrr: // ADD (register) T1
655   case ARM::tAND:   // AND (register) T1
656   case ARM::tASRri: // ASR (immediate) T1
657   case ARM::tASRrr: // ASR (register) T1
658   case ARM::tBIC:   // BIC (register) T1
659   case ARM::tEOR:   // EOR (register) T1
660   case ARM::tLSLri: // LSL (immediate) T1
661   case ARM::tLSLrr: // LSL (register) T1
662   case ARM::tLSRri: // LSR (immediate) T1
663   case ARM::tLSRrr: // LSR (register) T1
664   case ARM::tMUL:   // MUL T1
665   case ARM::tMVN:   // MVN (register) T1
666   case ARM::tORR:   // ORR (register) T1
667   case ARM::tROR:   // ROR (register) T1
668   case ARM::tRSB:   // RSB (immediate) T1
669   case ARM::tSBC:   // SBC (register) T1
670   case ARM::tSUBi3: // SUB (immediate) T1
671   case ARM::tSUBi8: // SUB (immediate) T2
672   case ARM::tSUBrr: // SUB (register) T1
673     return !ARMBaseInstrInfo::isCPSRDefined(*MI);
674   }
675 }
676 
677 /// isPredicable - Return true if the specified instruction can be predicated.
678 /// By default, this returns true for every instruction with a
679 /// PredicateOperand.
680 bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
681   if (!MI.isPredicable())
682     return false;
683 
684   if (MI.isBundle())
685     return false;
686 
687   if (!isEligibleForITBlock(&MI))
688     return false;
689 
690   const ARMFunctionInfo *AFI =
691       MI.getParent()->getParent()->getInfo<ARMFunctionInfo>();
692 
693   // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
694   // In their ARM encoding, they can't be encoded in a conditional form.
695   if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
696     return false;
697 
698   if (AFI->isThumb2Function()) {
699     if (getSubtarget().restrictIT())
700       return isV8EligibleForIT(&MI);
701   }
702 
703   return true;
704 }
705 
706 namespace llvm {
707 
708 template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
709   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
710     const MachineOperand &MO = MI->getOperand(i);
711     if (!MO.isReg() || MO.isUndef() || MO.isUse())
712       continue;
713     if (MO.getReg() != ARM::CPSR)
714       continue;
715     if (!MO.isDead())
716       return false;
717   }
718   // all definitions of CPSR are dead
719   return true;
720 }
721 
722 } // end namespace llvm
723 
724 /// GetInstSize - Return the size of the specified MachineInstr.
725 ///
726 unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
727   const MachineBasicBlock &MBB = *MI.getParent();
728   const MachineFunction *MF = MBB.getParent();
729   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
730 
731   const MCInstrDesc &MCID = MI.getDesc();
732   if (MCID.getSize())
733     return MCID.getSize();
734 
735   switch (MI.getOpcode()) {
736   default:
737     // pseudo-instruction sizes are zero.
738     return 0;
739   case TargetOpcode::BUNDLE:
740     return getInstBundleLength(MI);
741   case ARM::MOVi16_ga_pcrel:
742   case ARM::MOVTi16_ga_pcrel:
743   case ARM::t2MOVi16_ga_pcrel:
744   case ARM::t2MOVTi16_ga_pcrel:
745     return 4;
746   case ARM::MOVi32imm:
747   case ARM::t2MOVi32imm:
748     return 8;
749   case ARM::CONSTPOOL_ENTRY:
750   case ARM::JUMPTABLE_INSTS:
751   case ARM::JUMPTABLE_ADDRS:
752   case ARM::JUMPTABLE_TBB:
753   case ARM::JUMPTABLE_TBH:
754     // If this machine instr is a constant pool entry, its size is recorded as
755     // operand #2.
756     return MI.getOperand(2).getImm();
757   case ARM::Int_eh_sjlj_longjmp:
758     return 16;
759   case ARM::tInt_eh_sjlj_longjmp:
760     return 10;
761   case ARM::tInt_WIN_eh_sjlj_longjmp:
762     return 12;
763   case ARM::Int_eh_sjlj_setjmp:
764   case ARM::Int_eh_sjlj_setjmp_nofp:
765     return 20;
766   case ARM::tInt_eh_sjlj_setjmp:
767   case ARM::t2Int_eh_sjlj_setjmp:
768   case ARM::t2Int_eh_sjlj_setjmp_nofp:
769     return 12;
770   case ARM::SPACE:
771     return MI.getOperand(1).getImm();
772   case ARM::INLINEASM:
773   case ARM::INLINEASM_BR: {
774     // If this machine instr is an inline asm, measure it.
775     unsigned Size = getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
776     if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
777       Size = alignTo(Size, 4);
778     return Size;
779   }
780   }
781 }
782 
783 unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
784   unsigned Size = 0;
785   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
786   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
787   while (++I != E && I->isInsideBundle()) {
788     assert(!I->isBundle() && "No nested bundle!");
789     Size += getInstSizeInBytes(*I);
790   }
791   return Size;
792 }
793 
794 void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
795                                     MachineBasicBlock::iterator I,
796                                     unsigned DestReg, bool KillSrc,
797                                     const ARMSubtarget &Subtarget) const {
798   unsigned Opc = Subtarget.isThumb()
799                      ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
800                      : ARM::MRS;
801 
802   MachineInstrBuilder MIB =
803       BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg);
804 
805   // There is only 1 A/R class MRS instruction, and it always refers to
806   // APSR. However, there are lots of other possibilities on M-class cores.
807   if (Subtarget.isMClass())
808     MIB.addImm(0x800);
809 
810   MIB.add(predOps(ARMCC::AL))
811      .addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc));
812 }
813 
814 void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
815                                   MachineBasicBlock::iterator I,
816                                   unsigned SrcReg, bool KillSrc,
817                                   const ARMSubtarget &Subtarget) const {
818   unsigned Opc = Subtarget.isThumb()
819                      ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
820                      : ARM::MSR;
821 
822   MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc));
823 
824   if (Subtarget.isMClass())
825     MIB.addImm(0x800);
826   else
827     MIB.addImm(8);
828 
829   MIB.addReg(SrcReg, getKillRegState(KillSrc))
830      .add(predOps(ARMCC::AL))
831      .addReg(ARM::CPSR, RegState::Implicit | RegState::Define);
832 }
833 
834 void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
835   MIB.addImm(ARMVCC::None);
836   MIB.addReg(0);
837 }
838 
839 void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
840                                       Register DestReg) {
841   addUnpredicatedMveVpredNOp(MIB);
842   MIB.addReg(DestReg, RegState::Undef);
843 }
844 
845 void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
846   MIB.addImm(Cond);
847   MIB.addReg(ARM::VPR, RegState::Implicit);
848 }
849 
850 void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
851                                     unsigned Cond, unsigned Inactive) {
852   addPredicatedMveVpredNOp(MIB, Cond);
853   MIB.addReg(Inactive);
854 }
855 
856 void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
857                                    MachineBasicBlock::iterator I,
858                                    const DebugLoc &DL, MCRegister DestReg,
859                                    MCRegister SrcReg, bool KillSrc) const {
860   bool GPRDest = ARM::GPRRegClass.contains(DestReg);
861   bool GPRSrc = ARM::GPRRegClass.contains(SrcReg);
862 
863   if (GPRDest && GPRSrc) {
864     BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg)
865         .addReg(SrcReg, getKillRegState(KillSrc))
866         .add(predOps(ARMCC::AL))
867         .add(condCodeOp());
868     return;
869   }
870 
871   bool SPRDest = ARM::SPRRegClass.contains(DestReg);
872   bool SPRSrc = ARM::SPRRegClass.contains(SrcReg);
873 
874   unsigned Opc = 0;
875   if (SPRDest && SPRSrc)
876     Opc = ARM::VMOVS;
877   else if (GPRDest && SPRSrc)
878     Opc = ARM::VMOVRS;
879   else if (SPRDest && GPRSrc)
880     Opc = ARM::VMOVSR;
881   else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.hasFP64())
882     Opc = ARM::VMOVD;
883   else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
884     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
885 
886   if (Opc) {
887     MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
888     MIB.addReg(SrcReg, getKillRegState(KillSrc));
889     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
890       MIB.addReg(SrcReg, getKillRegState(KillSrc));
891     if (Opc == ARM::MVE_VORR)
892       addUnpredicatedMveVpredROp(MIB, DestReg);
893     else
894       MIB.add(predOps(ARMCC::AL));
895     return;
896   }
897 
898   // Handle register classes that require multiple instructions.
899   unsigned BeginIdx = 0;
900   unsigned SubRegs = 0;
901   int Spacing = 1;
902 
903   // Use VORRq when possible.
904   if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) {
905     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
906     BeginIdx = ARM::qsub_0;
907     SubRegs = 2;
908   } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
909     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
910     BeginIdx = ARM::qsub_0;
911     SubRegs = 4;
912   // Fall back to VMOVD.
913   } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) {
914     Opc = ARM::VMOVD;
915     BeginIdx = ARM::dsub_0;
916     SubRegs = 2;
917   } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) {
918     Opc = ARM::VMOVD;
919     BeginIdx = ARM::dsub_0;
920     SubRegs = 3;
921   } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) {
922     Opc = ARM::VMOVD;
923     BeginIdx = ARM::dsub_0;
924     SubRegs = 4;
925   } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) {
926     Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
927     BeginIdx = ARM::gsub_0;
928     SubRegs = 2;
929   } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) {
930     Opc = ARM::VMOVD;
931     BeginIdx = ARM::dsub_0;
932     SubRegs = 2;
933     Spacing = 2;
934   } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) {
935     Opc = ARM::VMOVD;
936     BeginIdx = ARM::dsub_0;
937     SubRegs = 3;
938     Spacing = 2;
939   } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) {
940     Opc = ARM::VMOVD;
941     BeginIdx = ARM::dsub_0;
942     SubRegs = 4;
943     Spacing = 2;
944   } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) &&
945              !Subtarget.hasFP64()) {
946     Opc = ARM::VMOVS;
947     BeginIdx = ARM::ssub_0;
948     SubRegs = 2;
949   } else if (SrcReg == ARM::CPSR) {
950     copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
951     return;
952   } else if (DestReg == ARM::CPSR) {
953     copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
954     return;
955   } else if (DestReg == ARM::VPR) {
956     assert(ARM::GPRRegClass.contains(SrcReg));
957     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_P0), DestReg)
958         .addReg(SrcReg, getKillRegState(KillSrc))
959         .add(predOps(ARMCC::AL));
960     return;
961   } else if (SrcReg == ARM::VPR) {
962     assert(ARM::GPRRegClass.contains(DestReg));
963     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_P0), DestReg)
964         .addReg(SrcReg, getKillRegState(KillSrc))
965         .add(predOps(ARMCC::AL));
966     return;
967   } else if (DestReg == ARM::FPSCR_NZCV) {
968     assert(ARM::GPRRegClass.contains(SrcReg));
969     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_FPSCR_NZCVQC), DestReg)
970         .addReg(SrcReg, getKillRegState(KillSrc))
971         .add(predOps(ARMCC::AL));
972     return;
973   } else if (SrcReg == ARM::FPSCR_NZCV) {
974     assert(ARM::GPRRegClass.contains(DestReg));
975     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_FPSCR_NZCVQC), DestReg)
976         .addReg(SrcReg, getKillRegState(KillSrc))
977         .add(predOps(ARMCC::AL));
978     return;
979   }
980 
981   assert(Opc && "Impossible reg-to-reg copy");
982 
983   const TargetRegisterInfo *TRI = &getRegisterInfo();
984   MachineInstrBuilder Mov;
985 
986   // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
987   if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
988     BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
989     Spacing = -Spacing;
990   }
991 #ifndef NDEBUG
992   SmallSet<unsigned, 4> DstRegs;
993 #endif
994   for (unsigned i = 0; i != SubRegs; ++i) {
995     Register Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
996     Register Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
997     assert(Dst && Src && "Bad sub-register");
998 #ifndef NDEBUG
999     assert(!DstRegs.count(Src) && "destructive vector copy");
1000     DstRegs.insert(Dst);
1001 #endif
1002     Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src);
1003     // VORR (NEON or MVE) takes two source operands.
1004     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
1005       Mov.addReg(Src);
1006     }
1007     // MVE VORR takes predicate operands in place of an ordinary condition.
1008     if (Opc == ARM::MVE_VORR)
1009       addUnpredicatedMveVpredROp(Mov, Dst);
1010     else
1011       Mov = Mov.add(predOps(ARMCC::AL));
1012     // MOVr can set CC.
1013     if (Opc == ARM::MOVr)
1014       Mov = Mov.add(condCodeOp());
1015   }
1016   // Add implicit super-register defs and kills to the last instruction.
1017   Mov->addRegisterDefined(DestReg, TRI);
1018   if (KillSrc)
1019     Mov->addRegisterKilled(SrcReg, TRI);
1020 }
1021 
1022 Optional<DestSourcePair>
1023 ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1024   // VMOVRRD is also a copy instruction but it requires
1025   // special way of handling. It is more complex copy version
1026   // and since that we are not considering it. For recognition
1027   // of such instruction isExtractSubregLike MI interface fuction
1028   // could be used.
1029   // VORRq is considered as a move only if two inputs are
1030   // the same register.
1031   if (!MI.isMoveReg() ||
1032       (MI.getOpcode() == ARM::VORRq &&
1033        MI.getOperand(1).getReg() != MI.getOperand(2).getReg()))
1034     return None;
1035   return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1036 }
1037 
1038 Optional<ParamLoadedValue>
1039 ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
1040                                       Register Reg) const {
1041   if (auto DstSrcPair = isCopyInstrImpl(MI)) {
1042     Register DstReg = DstSrcPair->Destination->getReg();
1043 
1044     // TODO: We don't handle cases where the forwarding reg is narrower/wider
1045     // than the copy registers. Consider for example:
1046     //
1047     //   s16 = VMOVS s0
1048     //   s17 = VMOVS s1
1049     //   call @callee(d0)
1050     //
1051     // We'd like to describe the call site value of d0 as d8, but this requires
1052     // gathering and merging the descriptions for the two VMOVS instructions.
1053     //
1054     // We also don't handle the reverse situation, where the forwarding reg is
1055     // narrower than the copy destination:
1056     //
1057     //   d8 = VMOVD d0
1058     //   call @callee(s1)
1059     //
1060     // We need to produce a fragment description (the call site value of s1 is
1061     // /not/ just d8).
1062     if (DstReg != Reg)
1063       return None;
1064   }
1065   return TargetInstrInfo::describeLoadedValue(MI, Reg);
1066 }
1067 
1068 const MachineInstrBuilder &
1069 ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
1070                           unsigned SubIdx, unsigned State,
1071                           const TargetRegisterInfo *TRI) const {
1072   if (!SubIdx)
1073     return MIB.addReg(Reg, State);
1074 
1075   if (Register::isPhysicalRegister(Reg))
1076     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1077   return MIB.addReg(Reg, State, SubIdx);
1078 }
1079 
1080 void ARMBaseInstrInfo::
1081 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1082                     Register SrcReg, bool isKill, int FI,
1083                     const TargetRegisterClass *RC,
1084                     const TargetRegisterInfo *TRI) const {
1085   MachineFunction &MF = *MBB.getParent();
1086   MachineFrameInfo &MFI = MF.getFrameInfo();
1087   Align Alignment = MFI.getObjectAlign(FI);
1088 
1089   MachineMemOperand *MMO = MF.getMachineMemOperand(
1090       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
1091       MFI.getObjectSize(FI), Alignment);
1092 
1093   switch (TRI->getSpillSize(*RC)) {
1094     case 2:
1095       if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1096         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRH))
1097             .addReg(SrcReg, getKillRegState(isKill))
1098             .addFrameIndex(FI)
1099             .addImm(0)
1100             .addMemOperand(MMO)
1101             .add(predOps(ARMCC::AL));
1102       } else
1103         llvm_unreachable("Unknown reg class!");
1104       break;
1105     case 4:
1106       if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1107         BuildMI(MBB, I, DebugLoc(), get(ARM::STRi12))
1108             .addReg(SrcReg, getKillRegState(isKill))
1109             .addFrameIndex(FI)
1110             .addImm(0)
1111             .addMemOperand(MMO)
1112             .add(predOps(ARMCC::AL));
1113       } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1114         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRS))
1115             .addReg(SrcReg, getKillRegState(isKill))
1116             .addFrameIndex(FI)
1117             .addImm(0)
1118             .addMemOperand(MMO)
1119             .add(predOps(ARMCC::AL));
1120       } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1121         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTR_P0_off))
1122             .addReg(SrcReg, getKillRegState(isKill))
1123             .addFrameIndex(FI)
1124             .addImm(0)
1125             .addMemOperand(MMO)
1126             .add(predOps(ARMCC::AL));
1127       } else
1128         llvm_unreachable("Unknown reg class!");
1129       break;
1130     case 8:
1131       if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1132         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRD))
1133             .addReg(SrcReg, getKillRegState(isKill))
1134             .addFrameIndex(FI)
1135             .addImm(0)
1136             .addMemOperand(MMO)
1137             .add(predOps(ARMCC::AL));
1138       } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1139         if (Subtarget.hasV5TEOps()) {
1140           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STRD));
1141           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1142           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1143           MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1144              .add(predOps(ARMCC::AL));
1145         } else {
1146           // Fallback to STM instruction, which has existed since the dawn of
1147           // time.
1148           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STMIA))
1149                                         .addFrameIndex(FI)
1150                                         .addMemOperand(MMO)
1151                                         .add(predOps(ARMCC::AL));
1152           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1153           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1154         }
1155       } else
1156         llvm_unreachable("Unknown reg class!");
1157       break;
1158     case 16:
1159       if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1160         // Use aligned spills if the stack can be realigned.
1161         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1162           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64))
1163               .addFrameIndex(FI)
1164               .addImm(16)
1165               .addReg(SrcReg, getKillRegState(isKill))
1166               .addMemOperand(MMO)
1167               .add(predOps(ARMCC::AL));
1168         } else {
1169           BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMQIA))
1170               .addReg(SrcReg, getKillRegState(isKill))
1171               .addFrameIndex(FI)
1172               .addMemOperand(MMO)
1173               .add(predOps(ARMCC::AL));
1174         }
1175       } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1176                  Subtarget.hasMVEIntegerOps()) {
1177         auto MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::MVE_VSTRWU32));
1178         MIB.addReg(SrcReg, getKillRegState(isKill))
1179           .addFrameIndex(FI)
1180           .addImm(0)
1181           .addMemOperand(MMO);
1182         addUnpredicatedMveVpredNOp(MIB);
1183       } else
1184         llvm_unreachable("Unknown reg class!");
1185       break;
1186     case 24:
1187       if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1188         // Use aligned spills if the stack can be realigned.
1189         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1190             Subtarget.hasNEON()) {
1191           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo))
1192               .addFrameIndex(FI)
1193               .addImm(16)
1194               .addReg(SrcReg, getKillRegState(isKill))
1195               .addMemOperand(MMO)
1196               .add(predOps(ARMCC::AL));
1197         } else {
1198           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1199                                             get(ARM::VSTMDIA))
1200                                         .addFrameIndex(FI)
1201                                         .add(predOps(ARMCC::AL))
1202                                         .addMemOperand(MMO);
1203           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1204           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1205           AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1206         }
1207       } else
1208         llvm_unreachable("Unknown reg class!");
1209       break;
1210     case 32:
1211       if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
1212         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1213             Subtarget.hasNEON()) {
1214           // FIXME: It's possible to only store part of the QQ register if the
1215           // spilled def has a sub-register index.
1216           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64QPseudo))
1217               .addFrameIndex(FI)
1218               .addImm(16)
1219               .addReg(SrcReg, getKillRegState(isKill))
1220               .addMemOperand(MMO)
1221               .add(predOps(ARMCC::AL));
1222         } else {
1223           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1224                                             get(ARM::VSTMDIA))
1225                                         .addFrameIndex(FI)
1226                                         .add(predOps(ARMCC::AL))
1227                                         .addMemOperand(MMO);
1228           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1229           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1230           MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1231                 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1232         }
1233       } else
1234         llvm_unreachable("Unknown reg class!");
1235       break;
1236     case 64:
1237       if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1238         MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMDIA))
1239                                       .addFrameIndex(FI)
1240                                       .add(predOps(ARMCC::AL))
1241                                       .addMemOperand(MMO);
1242         MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1243         MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1244         MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1245         MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1246         MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
1247         MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
1248         MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
1249               AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
1250       } else
1251         llvm_unreachable("Unknown reg class!");
1252       break;
1253     default:
1254       llvm_unreachable("Unknown reg class!");
1255   }
1256 }
1257 
1258 unsigned ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1259                                               int &FrameIndex) const {
1260   switch (MI.getOpcode()) {
1261   default: break;
1262   case ARM::STRrs:
1263   case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1264     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1265         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1266         MI.getOperand(3).getImm() == 0) {
1267       FrameIndex = MI.getOperand(1).getIndex();
1268       return MI.getOperand(0).getReg();
1269     }
1270     break;
1271   case ARM::STRi12:
1272   case ARM::t2STRi12:
1273   case ARM::tSTRspi:
1274   case ARM::VSTRD:
1275   case ARM::VSTRS:
1276     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1277         MI.getOperand(2).getImm() == 0) {
1278       FrameIndex = MI.getOperand(1).getIndex();
1279       return MI.getOperand(0).getReg();
1280     }
1281     break;
1282   case ARM::VSTR_P0_off:
1283     if (MI.getOperand(0).isFI() && MI.getOperand(1).isImm() &&
1284         MI.getOperand(1).getImm() == 0) {
1285       FrameIndex = MI.getOperand(0).getIndex();
1286       return ARM::P0;
1287     }
1288     break;
1289   case ARM::VST1q64:
1290   case ARM::VST1d64TPseudo:
1291   case ARM::VST1d64QPseudo:
1292     if (MI.getOperand(0).isFI() && MI.getOperand(2).getSubReg() == 0) {
1293       FrameIndex = MI.getOperand(0).getIndex();
1294       return MI.getOperand(2).getReg();
1295     }
1296     break;
1297   case ARM::VSTMQIA:
1298     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1299       FrameIndex = MI.getOperand(1).getIndex();
1300       return MI.getOperand(0).getReg();
1301     }
1302     break;
1303   }
1304 
1305   return 0;
1306 }
1307 
1308 unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1309                                                     int &FrameIndex) const {
1310   SmallVector<const MachineMemOperand *, 1> Accesses;
1311   if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1312       Accesses.size() == 1) {
1313     FrameIndex =
1314         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1315             ->getFrameIndex();
1316     return true;
1317   }
1318   return false;
1319 }
1320 
1321 void ARMBaseInstrInfo::
1322 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1323                      Register DestReg, int FI,
1324                      const TargetRegisterClass *RC,
1325                      const TargetRegisterInfo *TRI) const {
1326   DebugLoc DL;
1327   if (I != MBB.end()) DL = I->getDebugLoc();
1328   MachineFunction &MF = *MBB.getParent();
1329   MachineFrameInfo &MFI = MF.getFrameInfo();
1330   const Align Alignment = MFI.getObjectAlign(FI);
1331   MachineMemOperand *MMO = MF.getMachineMemOperand(
1332       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
1333       MFI.getObjectSize(FI), Alignment);
1334 
1335   switch (TRI->getSpillSize(*RC)) {
1336   case 2:
1337     if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1338       BuildMI(MBB, I, DL, get(ARM::VLDRH), DestReg)
1339           .addFrameIndex(FI)
1340           .addImm(0)
1341           .addMemOperand(MMO)
1342           .add(predOps(ARMCC::AL));
1343     } else
1344       llvm_unreachable("Unknown reg class!");
1345     break;
1346   case 4:
1347     if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1348       BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
1349           .addFrameIndex(FI)
1350           .addImm(0)
1351           .addMemOperand(MMO)
1352           .add(predOps(ARMCC::AL));
1353     } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1354       BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
1355           .addFrameIndex(FI)
1356           .addImm(0)
1357           .addMemOperand(MMO)
1358           .add(predOps(ARMCC::AL));
1359     } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1360       BuildMI(MBB, I, DL, get(ARM::VLDR_P0_off), DestReg)
1361           .addFrameIndex(FI)
1362           .addImm(0)
1363           .addMemOperand(MMO)
1364           .add(predOps(ARMCC::AL));
1365     } else
1366       llvm_unreachable("Unknown reg class!");
1367     break;
1368   case 8:
1369     if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1370       BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
1371           .addFrameIndex(FI)
1372           .addImm(0)
1373           .addMemOperand(MMO)
1374           .add(predOps(ARMCC::AL));
1375     } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1376       MachineInstrBuilder MIB;
1377 
1378       if (Subtarget.hasV5TEOps()) {
1379         MIB = BuildMI(MBB, I, DL, get(ARM::LDRD));
1380         AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1381         AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1382         MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1383            .add(predOps(ARMCC::AL));
1384       } else {
1385         // Fallback to LDM instruction, which has existed since the dawn of
1386         // time.
1387         MIB = BuildMI(MBB, I, DL, get(ARM::LDMIA))
1388                   .addFrameIndex(FI)
1389                   .addMemOperand(MMO)
1390                   .add(predOps(ARMCC::AL));
1391         MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1392         MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1393       }
1394 
1395       if (Register::isPhysicalRegister(DestReg))
1396         MIB.addReg(DestReg, RegState::ImplicitDefine);
1397     } else
1398       llvm_unreachable("Unknown reg class!");
1399     break;
1400   case 16:
1401     if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1402       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1403         BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
1404             .addFrameIndex(FI)
1405             .addImm(16)
1406             .addMemOperand(MMO)
1407             .add(predOps(ARMCC::AL));
1408       } else {
1409         BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
1410             .addFrameIndex(FI)
1411             .addMemOperand(MMO)
1412             .add(predOps(ARMCC::AL));
1413       }
1414     } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1415                Subtarget.hasMVEIntegerOps()) {
1416       auto MIB = BuildMI(MBB, I, DL, get(ARM::MVE_VLDRWU32), DestReg);
1417       MIB.addFrameIndex(FI)
1418         .addImm(0)
1419         .addMemOperand(MMO);
1420       addUnpredicatedMveVpredNOp(MIB);
1421     } else
1422       llvm_unreachable("Unknown reg class!");
1423     break;
1424   case 24:
1425     if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1426       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1427           Subtarget.hasNEON()) {
1428         BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
1429             .addFrameIndex(FI)
1430             .addImm(16)
1431             .addMemOperand(MMO)
1432             .add(predOps(ARMCC::AL));
1433       } else {
1434         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1435                                       .addFrameIndex(FI)
1436                                       .addMemOperand(MMO)
1437                                       .add(predOps(ARMCC::AL));
1438         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1439         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1440         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1441         if (Register::isPhysicalRegister(DestReg))
1442           MIB.addReg(DestReg, RegState::ImplicitDefine);
1443       }
1444     } else
1445       llvm_unreachable("Unknown reg class!");
1446     break;
1447    case 32:
1448     if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
1449       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1450           Subtarget.hasNEON()) {
1451         BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
1452             .addFrameIndex(FI)
1453             .addImm(16)
1454             .addMemOperand(MMO)
1455             .add(predOps(ARMCC::AL));
1456       } else {
1457         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1458                                       .addFrameIndex(FI)
1459                                       .add(predOps(ARMCC::AL))
1460                                       .addMemOperand(MMO);
1461         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1462         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1463         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1464         MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1465         if (Register::isPhysicalRegister(DestReg))
1466           MIB.addReg(DestReg, RegState::ImplicitDefine);
1467       }
1468     } else
1469       llvm_unreachable("Unknown reg class!");
1470     break;
1471   case 64:
1472     if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1473       MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1474                                     .addFrameIndex(FI)
1475                                     .add(predOps(ARMCC::AL))
1476                                     .addMemOperand(MMO);
1477       MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1478       MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1479       MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1480       MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1481       MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI);
1482       MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI);
1483       MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI);
1484       MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI);
1485       if (Register::isPhysicalRegister(DestReg))
1486         MIB.addReg(DestReg, RegState::ImplicitDefine);
1487     } else
1488       llvm_unreachable("Unknown reg class!");
1489     break;
1490   default:
1491     llvm_unreachable("Unknown regclass!");
1492   }
1493 }
1494 
1495 unsigned ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1496                                                int &FrameIndex) const {
1497   switch (MI.getOpcode()) {
1498   default: break;
1499   case ARM::LDRrs:
1500   case ARM::t2LDRs:  // FIXME: don't use t2LDRs to access frame.
1501     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1502         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1503         MI.getOperand(3).getImm() == 0) {
1504       FrameIndex = MI.getOperand(1).getIndex();
1505       return MI.getOperand(0).getReg();
1506     }
1507     break;
1508   case ARM::LDRi12:
1509   case ARM::t2LDRi12:
1510   case ARM::tLDRspi:
1511   case ARM::VLDRD:
1512   case ARM::VLDRS:
1513     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1514         MI.getOperand(2).getImm() == 0) {
1515       FrameIndex = MI.getOperand(1).getIndex();
1516       return MI.getOperand(0).getReg();
1517     }
1518     break;
1519   case ARM::VLDR_P0_off:
1520     if (MI.getOperand(0).isFI() && MI.getOperand(1).isImm() &&
1521         MI.getOperand(1).getImm() == 0) {
1522       FrameIndex = MI.getOperand(0).getIndex();
1523       return ARM::P0;
1524     }
1525     break;
1526   case ARM::VLD1q64:
1527   case ARM::VLD1d8TPseudo:
1528   case ARM::VLD1d16TPseudo:
1529   case ARM::VLD1d32TPseudo:
1530   case ARM::VLD1d64TPseudo:
1531   case ARM::VLD1d8QPseudo:
1532   case ARM::VLD1d16QPseudo:
1533   case ARM::VLD1d32QPseudo:
1534   case ARM::VLD1d64QPseudo:
1535     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1536       FrameIndex = MI.getOperand(1).getIndex();
1537       return MI.getOperand(0).getReg();
1538     }
1539     break;
1540   case ARM::VLDMQIA:
1541     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1542       FrameIndex = MI.getOperand(1).getIndex();
1543       return MI.getOperand(0).getReg();
1544     }
1545     break;
1546   }
1547 
1548   return 0;
1549 }
1550 
1551 unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1552                                                      int &FrameIndex) const {
1553   SmallVector<const MachineMemOperand *, 1> Accesses;
1554   if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1555       Accesses.size() == 1) {
1556     FrameIndex =
1557         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1558             ->getFrameIndex();
1559     return true;
1560   }
1561   return false;
1562 }
1563 
1564 /// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1565 /// depending on whether the result is used.
1566 void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1567   bool isThumb1 = Subtarget.isThumb1Only();
1568   bool isThumb2 = Subtarget.isThumb2();
1569   const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1570 
1571   DebugLoc dl = MI->getDebugLoc();
1572   MachineBasicBlock *BB = MI->getParent();
1573 
1574   MachineInstrBuilder LDM, STM;
1575   if (isThumb1 || !MI->getOperand(1).isDead()) {
1576     MachineOperand LDWb(MI->getOperand(1));
1577     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA_UPD
1578                                                  : isThumb1 ? ARM::tLDMIA_UPD
1579                                                             : ARM::LDMIA_UPD))
1580               .add(LDWb);
1581   } else {
1582     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1583   }
1584 
1585   if (isThumb1 || !MI->getOperand(0).isDead()) {
1586     MachineOperand STWb(MI->getOperand(0));
1587     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA_UPD
1588                                                  : isThumb1 ? ARM::tSTMIA_UPD
1589                                                             : ARM::STMIA_UPD))
1590               .add(STWb);
1591   } else {
1592     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1593   }
1594 
1595   MachineOperand LDBase(MI->getOperand(3));
1596   LDM.add(LDBase).add(predOps(ARMCC::AL));
1597 
1598   MachineOperand STBase(MI->getOperand(2));
1599   STM.add(STBase).add(predOps(ARMCC::AL));
1600 
1601   // Sort the scratch registers into ascending order.
1602   const TargetRegisterInfo &TRI = getRegisterInfo();
1603   SmallVector<unsigned, 6> ScratchRegs;
1604   for(unsigned I = 5; I < MI->getNumOperands(); ++I)
1605     ScratchRegs.push_back(MI->getOperand(I).getReg());
1606   llvm::sort(ScratchRegs,
1607              [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1608                return TRI.getEncodingValue(Reg1) <
1609                       TRI.getEncodingValue(Reg2);
1610              });
1611 
1612   for (const auto &Reg : ScratchRegs) {
1613     LDM.addReg(Reg, RegState::Define);
1614     STM.addReg(Reg, RegState::Kill);
1615   }
1616 
1617   BB->erase(MI);
1618 }
1619 
1620 bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1621   if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1622     assert(getSubtarget().getTargetTriple().isOSBinFormatMachO() &&
1623            "LOAD_STACK_GUARD currently supported only for MachO.");
1624     expandLoadStackGuard(MI);
1625     MI.getParent()->erase(MI);
1626     return true;
1627   }
1628 
1629   if (MI.getOpcode() == ARM::MEMCPY) {
1630     expandMEMCPY(MI);
1631     return true;
1632   }
1633 
1634   // This hook gets to expand COPY instructions before they become
1635   // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
1636   // widened to VMOVD.  We prefer the VMOVD when possible because it may be
1637   // changed into a VORR that can go down the NEON pipeline.
1638   if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1639     return false;
1640 
1641   // Look for a copy between even S-registers.  That is where we keep floats
1642   // when using NEON v2f32 instructions for f32 arithmetic.
1643   Register DstRegS = MI.getOperand(0).getReg();
1644   Register SrcRegS = MI.getOperand(1).getReg();
1645   if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
1646     return false;
1647 
1648   const TargetRegisterInfo *TRI = &getRegisterInfo();
1649   unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
1650                                               &ARM::DPRRegClass);
1651   unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
1652                                               &ARM::DPRRegClass);
1653   if (!DstRegD || !SrcRegD)
1654     return false;
1655 
1656   // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
1657   // legal if the COPY already defines the full DstRegD, and it isn't a
1658   // sub-register insertion.
1659   if (!MI.definesRegister(DstRegD, TRI) || MI.readsRegister(DstRegD, TRI))
1660     return false;
1661 
1662   // A dead copy shouldn't show up here, but reject it just in case.
1663   if (MI.getOperand(0).isDead())
1664     return false;
1665 
1666   // All clear, widen the COPY.
1667   LLVM_DEBUG(dbgs() << "widening:    " << MI);
1668   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1669 
1670   // Get rid of the old implicit-def of DstRegD.  Leave it if it defines a Q-reg
1671   // or some other super-register.
1672   int ImpDefIdx = MI.findRegisterDefOperandIdx(DstRegD);
1673   if (ImpDefIdx != -1)
1674     MI.RemoveOperand(ImpDefIdx);
1675 
1676   // Change the opcode and operands.
1677   MI.setDesc(get(ARM::VMOVD));
1678   MI.getOperand(0).setReg(DstRegD);
1679   MI.getOperand(1).setReg(SrcRegD);
1680   MIB.add(predOps(ARMCC::AL));
1681 
1682   // We are now reading SrcRegD instead of SrcRegS.  This may upset the
1683   // register scavenger and machine verifier, so we need to indicate that we
1684   // are reading an undefined value from SrcRegD, but a proper value from
1685   // SrcRegS.
1686   MI.getOperand(1).setIsUndef();
1687   MIB.addReg(SrcRegS, RegState::Implicit);
1688 
1689   // SrcRegD may actually contain an unrelated value in the ssub_1
1690   // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
1691   if (MI.getOperand(1).isKill()) {
1692     MI.getOperand(1).setIsKill(false);
1693     MI.addRegisterKilled(SrcRegS, TRI, true);
1694   }
1695 
1696   LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1697   return true;
1698 }
1699 
1700 /// Create a copy of a const pool value. Update CPI to the new index and return
1701 /// the label UID.
1702 static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1703   MachineConstantPool *MCP = MF.getConstantPool();
1704   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1705 
1706   const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1707   assert(MCPE.isMachineConstantPoolEntry() &&
1708          "Expecting a machine constantpool entry!");
1709   ARMConstantPoolValue *ACPV =
1710     static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1711 
1712   unsigned PCLabelId = AFI->createPICLabelUId();
1713   ARMConstantPoolValue *NewCPV = nullptr;
1714 
1715   // FIXME: The below assumes PIC relocation model and that the function
1716   // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1717   // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1718   // instructions, so that's probably OK, but is PIC always correct when
1719   // we get here?
1720   if (ACPV->isGlobalValue())
1721     NewCPV = ARMConstantPoolConstant::Create(
1722         cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, ARMCP::CPValue,
1723         4, ACPV->getModifier(), ACPV->mustAddCurrentAddress());
1724   else if (ACPV->isExtSymbol())
1725     NewCPV = ARMConstantPoolSymbol::
1726       Create(MF.getFunction().getContext(),
1727              cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
1728   else if (ACPV->isBlockAddress())
1729     NewCPV = ARMConstantPoolConstant::
1730       Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
1731              ARMCP::CPBlockAddress, 4);
1732   else if (ACPV->isLSDA())
1733     NewCPV = ARMConstantPoolConstant::Create(&MF.getFunction(), PCLabelId,
1734                                              ARMCP::CPLSDA, 4);
1735   else if (ACPV->isMachineBasicBlock())
1736     NewCPV = ARMConstantPoolMBB::
1737       Create(MF.getFunction().getContext(),
1738              cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
1739   else
1740     llvm_unreachable("Unexpected ARM constantpool value type!!");
1741   CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlign());
1742   return PCLabelId;
1743 }
1744 
1745 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1746                                      MachineBasicBlock::iterator I,
1747                                      Register DestReg, unsigned SubIdx,
1748                                      const MachineInstr &Orig,
1749                                      const TargetRegisterInfo &TRI) const {
1750   unsigned Opcode = Orig.getOpcode();
1751   switch (Opcode) {
1752   default: {
1753     MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
1754     MI->substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1755     MBB.insert(I, MI);
1756     break;
1757   }
1758   case ARM::tLDRpci_pic:
1759   case ARM::t2LDRpci_pic: {
1760     MachineFunction &MF = *MBB.getParent();
1761     unsigned CPI = Orig.getOperand(1).getIndex();
1762     unsigned PCLabelId = duplicateCPV(MF, CPI);
1763     BuildMI(MBB, I, Orig.getDebugLoc(), get(Opcode), DestReg)
1764         .addConstantPoolIndex(CPI)
1765         .addImm(PCLabelId)
1766         .cloneMemRefs(Orig);
1767     break;
1768   }
1769   }
1770 }
1771 
1772 MachineInstr &
1773 ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1774     MachineBasicBlock::iterator InsertBefore,
1775     const MachineInstr &Orig) const {
1776   MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1777   MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1778   for (;;) {
1779     switch (I->getOpcode()) {
1780     case ARM::tLDRpci_pic:
1781     case ARM::t2LDRpci_pic: {
1782       MachineFunction &MF = *MBB.getParent();
1783       unsigned CPI = I->getOperand(1).getIndex();
1784       unsigned PCLabelId = duplicateCPV(MF, CPI);
1785       I->getOperand(1).setIndex(CPI);
1786       I->getOperand(2).setImm(PCLabelId);
1787       break;
1788     }
1789     }
1790     if (!I->isBundledWithSucc())
1791       break;
1792     ++I;
1793   }
1794   return Cloned;
1795 }
1796 
1797 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1798                                         const MachineInstr &MI1,
1799                                         const MachineRegisterInfo *MRI) const {
1800   unsigned Opcode = MI0.getOpcode();
1801   if (Opcode == ARM::t2LDRpci ||
1802       Opcode == ARM::t2LDRpci_pic ||
1803       Opcode == ARM::tLDRpci ||
1804       Opcode == ARM::tLDRpci_pic ||
1805       Opcode == ARM::LDRLIT_ga_pcrel ||
1806       Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1807       Opcode == ARM::tLDRLIT_ga_pcrel ||
1808       Opcode == ARM::MOV_ga_pcrel ||
1809       Opcode == ARM::MOV_ga_pcrel_ldr ||
1810       Opcode == ARM::t2MOV_ga_pcrel) {
1811     if (MI1.getOpcode() != Opcode)
1812       return false;
1813     if (MI0.getNumOperands() != MI1.getNumOperands())
1814       return false;
1815 
1816     const MachineOperand &MO0 = MI0.getOperand(1);
1817     const MachineOperand &MO1 = MI1.getOperand(1);
1818     if (MO0.getOffset() != MO1.getOffset())
1819       return false;
1820 
1821     if (Opcode == ARM::LDRLIT_ga_pcrel ||
1822         Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1823         Opcode == ARM::tLDRLIT_ga_pcrel ||
1824         Opcode == ARM::MOV_ga_pcrel ||
1825         Opcode == ARM::MOV_ga_pcrel_ldr ||
1826         Opcode == ARM::t2MOV_ga_pcrel)
1827       // Ignore the PC labels.
1828       return MO0.getGlobal() == MO1.getGlobal();
1829 
1830     const MachineFunction *MF = MI0.getParent()->getParent();
1831     const MachineConstantPool *MCP = MF->getConstantPool();
1832     int CPI0 = MO0.getIndex();
1833     int CPI1 = MO1.getIndex();
1834     const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1835     const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1836     bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1837     bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1838     if (isARMCP0 && isARMCP1) {
1839       ARMConstantPoolValue *ACPV0 =
1840         static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1841       ARMConstantPoolValue *ACPV1 =
1842         static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1843       return ACPV0->hasSameValue(ACPV1);
1844     } else if (!isARMCP0 && !isARMCP1) {
1845       return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1846     }
1847     return false;
1848   } else if (Opcode == ARM::PICLDR) {
1849     if (MI1.getOpcode() != Opcode)
1850       return false;
1851     if (MI0.getNumOperands() != MI1.getNumOperands())
1852       return false;
1853 
1854     Register Addr0 = MI0.getOperand(1).getReg();
1855     Register Addr1 = MI1.getOperand(1).getReg();
1856     if (Addr0 != Addr1) {
1857       if (!MRI || !Register::isVirtualRegister(Addr0) ||
1858           !Register::isVirtualRegister(Addr1))
1859         return false;
1860 
1861       // This assumes SSA form.
1862       MachineInstr *Def0 = MRI->getVRegDef(Addr0);
1863       MachineInstr *Def1 = MRI->getVRegDef(Addr1);
1864       // Check if the loaded value, e.g. a constantpool of a global address, are
1865       // the same.
1866       if (!produceSameValue(*Def0, *Def1, MRI))
1867         return false;
1868     }
1869 
1870     for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1871       // %12 = PICLDR %11, 0, 14, %noreg
1872       const MachineOperand &MO0 = MI0.getOperand(i);
1873       const MachineOperand &MO1 = MI1.getOperand(i);
1874       if (!MO0.isIdenticalTo(MO1))
1875         return false;
1876     }
1877     return true;
1878   }
1879 
1880   return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
1881 }
1882 
1883 /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1884 /// determine if two loads are loading from the same base address. It should
1885 /// only return true if the base pointers are the same and the only differences
1886 /// between the two addresses is the offset. It also returns the offsets by
1887 /// reference.
1888 ///
1889 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1890 /// is permanently disabled.
1891 bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1892                                                int64_t &Offset1,
1893                                                int64_t &Offset2) const {
1894   // Don't worry about Thumb: just ARM and Thumb2.
1895   if (Subtarget.isThumb1Only()) return false;
1896 
1897   if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1898     return false;
1899 
1900   switch (Load1->getMachineOpcode()) {
1901   default:
1902     return false;
1903   case ARM::LDRi12:
1904   case ARM::LDRBi12:
1905   case ARM::LDRD:
1906   case ARM::LDRH:
1907   case ARM::LDRSB:
1908   case ARM::LDRSH:
1909   case ARM::VLDRD:
1910   case ARM::VLDRS:
1911   case ARM::t2LDRi8:
1912   case ARM::t2LDRBi8:
1913   case ARM::t2LDRDi8:
1914   case ARM::t2LDRSHi8:
1915   case ARM::t2LDRi12:
1916   case ARM::t2LDRBi12:
1917   case ARM::t2LDRSHi12:
1918     break;
1919   }
1920 
1921   switch (Load2->getMachineOpcode()) {
1922   default:
1923     return false;
1924   case ARM::LDRi12:
1925   case ARM::LDRBi12:
1926   case ARM::LDRD:
1927   case ARM::LDRH:
1928   case ARM::LDRSB:
1929   case ARM::LDRSH:
1930   case ARM::VLDRD:
1931   case ARM::VLDRS:
1932   case ARM::t2LDRi8:
1933   case ARM::t2LDRBi8:
1934   case ARM::t2LDRSHi8:
1935   case ARM::t2LDRi12:
1936   case ARM::t2LDRBi12:
1937   case ARM::t2LDRSHi12:
1938     break;
1939   }
1940 
1941   // Check if base addresses and chain operands match.
1942   if (Load1->getOperand(0) != Load2->getOperand(0) ||
1943       Load1->getOperand(4) != Load2->getOperand(4))
1944     return false;
1945 
1946   // Index should be Reg0.
1947   if (Load1->getOperand(3) != Load2->getOperand(3))
1948     return false;
1949 
1950   // Determine the offsets.
1951   if (isa<ConstantSDNode>(Load1->getOperand(1)) &&
1952       isa<ConstantSDNode>(Load2->getOperand(1))) {
1953     Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue();
1954     Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue();
1955     return true;
1956   }
1957 
1958   return false;
1959 }
1960 
1961 /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1962 /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1963 /// be scheduled togther. On some targets if two loads are loading from
1964 /// addresses in the same cache line, it's better if they are scheduled
1965 /// together. This function takes two integers that represent the load offsets
1966 /// from the common base address. It returns true if it decides it's desirable
1967 /// to schedule the two loads together. "NumLoads" is the number of loads that
1968 /// have already been scheduled after Load1.
1969 ///
1970 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1971 /// is permanently disabled.
1972 bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1973                                                int64_t Offset1, int64_t Offset2,
1974                                                unsigned NumLoads) const {
1975   // Don't worry about Thumb: just ARM and Thumb2.
1976   if (Subtarget.isThumb1Only()) return false;
1977 
1978   assert(Offset2 > Offset1);
1979 
1980   if ((Offset2 - Offset1) / 8 > 64)
1981     return false;
1982 
1983   // Check if the machine opcodes are different. If they are different
1984   // then we consider them to not be of the same base address,
1985   // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
1986   // In this case, they are considered to be the same because they are different
1987   // encoding forms of the same basic instruction.
1988   if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
1989       !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
1990          Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
1991         (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
1992          Load2->getMachineOpcode() == ARM::t2LDRBi8)))
1993     return false;  // FIXME: overly conservative?
1994 
1995   // Four loads in a row should be sufficient.
1996   if (NumLoads >= 3)
1997     return false;
1998 
1999   return true;
2000 }
2001 
2002 bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2003                                             const MachineBasicBlock *MBB,
2004                                             const MachineFunction &MF) const {
2005   // Debug info is never a scheduling boundary. It's necessary to be explicit
2006   // due to the special treatment of IT instructions below, otherwise a
2007   // dbg_value followed by an IT will result in the IT instruction being
2008   // considered a scheduling hazard, which is wrong. It should be the actual
2009   // instruction preceding the dbg_value instruction(s), just like it is
2010   // when debug info is not present.
2011   if (MI.isDebugInstr())
2012     return false;
2013 
2014   // Terminators and labels can't be scheduled around.
2015   if (MI.isTerminator() || MI.isPosition())
2016     return true;
2017 
2018   // Treat the start of the IT block as a scheduling boundary, but schedule
2019   // t2IT along with all instructions following it.
2020   // FIXME: This is a big hammer. But the alternative is to add all potential
2021   // true and anti dependencies to IT block instructions as implicit operands
2022   // to the t2IT instruction. The added compile time and complexity does not
2023   // seem worth it.
2024   MachineBasicBlock::const_iterator I = MI;
2025   // Make sure to skip any debug instructions
2026   while (++I != MBB->end() && I->isDebugInstr())
2027     ;
2028   if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
2029     return true;
2030 
2031   // Don't attempt to schedule around any instruction that defines
2032   // a stack-oriented pointer, as it's unlikely to be profitable. This
2033   // saves compile time, because it doesn't require every single
2034   // stack slot reference to depend on the instruction that does the
2035   // modification.
2036   // Calls don't actually change the stack pointer, even if they have imp-defs.
2037   // No ARM calling conventions change the stack pointer. (X86 calling
2038   // conventions sometimes do).
2039   if (!MI.isCall() && MI.definesRegister(ARM::SP))
2040     return true;
2041 
2042   return false;
2043 }
2044 
2045 bool ARMBaseInstrInfo::
2046 isProfitableToIfCvt(MachineBasicBlock &MBB,
2047                     unsigned NumCycles, unsigned ExtraPredCycles,
2048                     BranchProbability Probability) const {
2049   if (!NumCycles)
2050     return false;
2051 
2052   // If we are optimizing for size, see if the branch in the predecessor can be
2053   // lowered to cbn?z by the constant island lowering pass, and return false if
2054   // so. This results in a shorter instruction sequence.
2055   if (MBB.getParent()->getFunction().hasOptSize()) {
2056     MachineBasicBlock *Pred = *MBB.pred_begin();
2057     if (!Pred->empty()) {
2058       MachineInstr *LastMI = &*Pred->rbegin();
2059       if (LastMI->getOpcode() == ARM::t2Bcc) {
2060         const TargetRegisterInfo *TRI = &getRegisterInfo();
2061         MachineInstr *CmpMI = findCMPToFoldIntoCBZ(LastMI, TRI);
2062         if (CmpMI)
2063           return false;
2064       }
2065     }
2066   }
2067   return isProfitableToIfCvt(MBB, NumCycles, ExtraPredCycles,
2068                              MBB, 0, 0, Probability);
2069 }
2070 
2071 bool ARMBaseInstrInfo::
2072 isProfitableToIfCvt(MachineBasicBlock &TBB,
2073                     unsigned TCycles, unsigned TExtra,
2074                     MachineBasicBlock &FBB,
2075                     unsigned FCycles, unsigned FExtra,
2076                     BranchProbability Probability) const {
2077   if (!TCycles)
2078     return false;
2079 
2080   // In thumb code we often end up trading one branch for a IT block, and
2081   // if we are cloning the instruction can increase code size. Prevent
2082   // blocks with multiple predecesors from being ifcvted to prevent this
2083   // cloning.
2084   if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
2085     if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
2086       return false;
2087   }
2088 
2089   // Attempt to estimate the relative costs of predication versus branching.
2090   // Here we scale up each component of UnpredCost to avoid precision issue when
2091   // scaling TCycles/FCycles by Probability.
2092   const unsigned ScalingUpFactor = 1024;
2093 
2094   unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
2095   unsigned UnpredCost;
2096   if (!Subtarget.hasBranchPredictor()) {
2097     // When we don't have a branch predictor it's always cheaper to not take a
2098     // branch than take it, so we have to take that into account.
2099     unsigned NotTakenBranchCost = 1;
2100     unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
2101     unsigned TUnpredCycles, FUnpredCycles;
2102     if (!FCycles) {
2103       // Triangle: TBB is the fallthrough
2104       TUnpredCycles = TCycles + NotTakenBranchCost;
2105       FUnpredCycles = TakenBranchCost;
2106     } else {
2107       // Diamond: TBB is the block that is branched to, FBB is the fallthrough
2108       TUnpredCycles = TCycles + TakenBranchCost;
2109       FUnpredCycles = FCycles + NotTakenBranchCost;
2110       // The branch at the end of FBB will disappear when it's predicated, so
2111       // discount it from PredCost.
2112       PredCost -= 1 * ScalingUpFactor;
2113     }
2114     // The total cost is the cost of each path scaled by their probabilites
2115     unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor);
2116     unsigned FUnpredCost = Probability.getCompl().scale(FUnpredCycles * ScalingUpFactor);
2117     UnpredCost = TUnpredCost + FUnpredCost;
2118     // When predicating assume that the first IT can be folded away but later
2119     // ones cost one cycle each
2120     if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2121       PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2122     }
2123   } else {
2124     unsigned TUnpredCost = Probability.scale(TCycles * ScalingUpFactor);
2125     unsigned FUnpredCost =
2126       Probability.getCompl().scale(FCycles * ScalingUpFactor);
2127     UnpredCost = TUnpredCost + FUnpredCost;
2128     UnpredCost += 1 * ScalingUpFactor; // The branch itself
2129     UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2130   }
2131 
2132   return PredCost <= UnpredCost;
2133 }
2134 
2135 unsigned
2136 ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2137                                                    unsigned NumInsts) const {
2138   // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2139   // ARM has a condition code field in every predicable instruction, using it
2140   // doesn't change code size.
2141   return Subtarget.isThumb2() ? divideCeil(NumInsts, 4) * 2 : 0;
2142 }
2143 
2144 unsigned
2145 ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2146   // If this branch is likely to be folded into the comparison to form a
2147   // CB(N)Z, then removing it won't reduce code size at all, because that will
2148   // just replace the CB(N)Z with a CMP.
2149   if (MI.getOpcode() == ARM::t2Bcc &&
2150       findCMPToFoldIntoCBZ(&MI, &getRegisterInfo()))
2151     return 0;
2152 
2153   unsigned Size = getInstSizeInBytes(MI);
2154 
2155   // For Thumb2, all branches are 32-bit instructions during the if conversion
2156   // pass, but may be replaced with 16-bit instructions during size reduction.
2157   // Since the branches considered by if conversion tend to be forward branches
2158   // over small basic blocks, they are very likely to be in range for the
2159   // narrow instructions, so we assume the final code size will be half what it
2160   // currently is.
2161   if (Subtarget.isThumb2())
2162     Size /= 2;
2163 
2164   return Size;
2165 }
2166 
2167 bool
2168 ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2169                                             MachineBasicBlock &FMBB) const {
2170   // Reduce false anti-dependencies to let the target's out-of-order execution
2171   // engine do its thing.
2172   return Subtarget.isProfitableToUnpredicate();
2173 }
2174 
2175 /// getInstrPredicate - If instruction is predicated, returns its predicate
2176 /// condition, otherwise returns AL. It also returns the condition code
2177 /// register by reference.
2178 ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2179                                          Register &PredReg) {
2180   int PIdx = MI.findFirstPredOperandIdx();
2181   if (PIdx == -1) {
2182     PredReg = 0;
2183     return ARMCC::AL;
2184   }
2185 
2186   PredReg = MI.getOperand(PIdx+1).getReg();
2187   return (ARMCC::CondCodes)MI.getOperand(PIdx).getImm();
2188 }
2189 
2190 unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2191   if (Opc == ARM::B)
2192     return ARM::Bcc;
2193   if (Opc == ARM::tB)
2194     return ARM::tBcc;
2195   if (Opc == ARM::t2B)
2196     return ARM::t2Bcc;
2197 
2198   llvm_unreachable("Unknown unconditional branch opcode!");
2199 }
2200 
2201 MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2202                                                        bool NewMI,
2203                                                        unsigned OpIdx1,
2204                                                        unsigned OpIdx2) const {
2205   switch (MI.getOpcode()) {
2206   case ARM::MOVCCr:
2207   case ARM::t2MOVCCr: {
2208     // MOVCC can be commuted by inverting the condition.
2209     Register PredReg;
2210     ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2211     // MOVCC AL can't be inverted. Shouldn't happen.
2212     if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2213       return nullptr;
2214     MachineInstr *CommutedMI =
2215         TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2216     if (!CommutedMI)
2217       return nullptr;
2218     // After swapping the MOVCC operands, also invert the condition.
2219     CommutedMI->getOperand(CommutedMI->findFirstPredOperandIdx())
2220         .setImm(ARMCC::getOppositeCondition(CC));
2221     return CommutedMI;
2222   }
2223   }
2224   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2225 }
2226 
2227 /// Identify instructions that can be folded into a MOVCC instruction, and
2228 /// return the defining instruction.
2229 MachineInstr *
2230 ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2231                                    const TargetInstrInfo *TII) const {
2232   if (!Reg.isVirtual())
2233     return nullptr;
2234   if (!MRI.hasOneNonDBGUse(Reg))
2235     return nullptr;
2236   MachineInstr *MI = MRI.getVRegDef(Reg);
2237   if (!MI)
2238     return nullptr;
2239   // Check if MI can be predicated and folded into the MOVCC.
2240   if (!isPredicable(*MI))
2241     return nullptr;
2242   // Check if MI has any non-dead defs or physreg uses. This also detects
2243   // predicated instructions which will be reading CPSR.
2244   for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
2245     const MachineOperand &MO = MI->getOperand(i);
2246     // Reject frame index operands, PEI can't handle the predicated pseudos.
2247     if (MO.isFI() || MO.isCPI() || MO.isJTI())
2248       return nullptr;
2249     if (!MO.isReg())
2250       continue;
2251     // MI can't have any tied operands, that would conflict with predication.
2252     if (MO.isTied())
2253       return nullptr;
2254     if (Register::isPhysicalRegister(MO.getReg()))
2255       return nullptr;
2256     if (MO.isDef() && !MO.isDead())
2257       return nullptr;
2258   }
2259   bool DontMoveAcrossStores = true;
2260   if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
2261     return nullptr;
2262   return MI;
2263 }
2264 
2265 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr &MI,
2266                                      SmallVectorImpl<MachineOperand> &Cond,
2267                                      unsigned &TrueOp, unsigned &FalseOp,
2268                                      bool &Optimizable) const {
2269   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2270          "Unknown select instruction");
2271   // MOVCC operands:
2272   // 0: Def.
2273   // 1: True use.
2274   // 2: False use.
2275   // 3: Condition code.
2276   // 4: CPSR use.
2277   TrueOp = 1;
2278   FalseOp = 2;
2279   Cond.push_back(MI.getOperand(3));
2280   Cond.push_back(MI.getOperand(4));
2281   // We can always fold a def.
2282   Optimizable = true;
2283   return false;
2284 }
2285 
2286 MachineInstr *
2287 ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2288                                  SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2289                                  bool PreferFalse) const {
2290   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2291          "Unknown select instruction");
2292   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2293   MachineInstr *DefMI = canFoldIntoMOVCC(MI.getOperand(2).getReg(), MRI, this);
2294   bool Invert = !DefMI;
2295   if (!DefMI)
2296     DefMI = canFoldIntoMOVCC(MI.getOperand(1).getReg(), MRI, this);
2297   if (!DefMI)
2298     return nullptr;
2299 
2300   // Find new register class to use.
2301   MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1);
2302   Register DestReg = MI.getOperand(0).getReg();
2303   const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
2304   if (!MRI.constrainRegClass(DestReg, PreviousClass))
2305     return nullptr;
2306 
2307   // Create a new predicated version of DefMI.
2308   // Rfalse is the first use.
2309   MachineInstrBuilder NewMI =
2310       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), DefMI->getDesc(), DestReg);
2311 
2312   // Copy all the DefMI operands, excluding its (null) predicate.
2313   const MCInstrDesc &DefDesc = DefMI->getDesc();
2314   for (unsigned i = 1, e = DefDesc.getNumOperands();
2315        i != e && !DefDesc.OpInfo[i].isPredicate(); ++i)
2316     NewMI.add(DefMI->getOperand(i));
2317 
2318   unsigned CondCode = MI.getOperand(3).getImm();
2319   if (Invert)
2320     NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode)));
2321   else
2322     NewMI.addImm(CondCode);
2323   NewMI.add(MI.getOperand(4));
2324 
2325   // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2326   if (NewMI->hasOptionalDef())
2327     NewMI.add(condCodeOp());
2328 
2329   // The output register value when the predicate is false is an implicit
2330   // register operand tied to the first def.
2331   // The tie makes the register allocator ensure the FalseReg is allocated the
2332   // same register as operand 0.
2333   FalseReg.setImplicit();
2334   NewMI.add(FalseReg);
2335   NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
2336 
2337   // Update SeenMIs set: register newly created MI and erase removed DefMI.
2338   SeenMIs.insert(NewMI);
2339   SeenMIs.erase(DefMI);
2340 
2341   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2342   // DefMI would be invalid when tranferred inside the loop.  Checking for a
2343   // loop is expensive, but at least remove kill flags if they are in different
2344   // BBs.
2345   if (DefMI->getParent() != MI.getParent())
2346     NewMI->clearKillInfo();
2347 
2348   // The caller will erase MI, but not DefMI.
2349   DefMI->eraseFromParent();
2350   return NewMI;
2351 }
2352 
2353 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2354 /// instruction is encoded with an 'S' bit is determined by the optional CPSR
2355 /// def operand.
2356 ///
2357 /// This will go away once we can teach tblgen how to set the optional CPSR def
2358 /// operand itself.
2359 struct AddSubFlagsOpcodePair {
2360   uint16_t PseudoOpc;
2361   uint16_t MachineOpc;
2362 };
2363 
2364 static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2365   {ARM::ADDSri, ARM::ADDri},
2366   {ARM::ADDSrr, ARM::ADDrr},
2367   {ARM::ADDSrsi, ARM::ADDrsi},
2368   {ARM::ADDSrsr, ARM::ADDrsr},
2369 
2370   {ARM::SUBSri, ARM::SUBri},
2371   {ARM::SUBSrr, ARM::SUBrr},
2372   {ARM::SUBSrsi, ARM::SUBrsi},
2373   {ARM::SUBSrsr, ARM::SUBrsr},
2374 
2375   {ARM::RSBSri, ARM::RSBri},
2376   {ARM::RSBSrsi, ARM::RSBrsi},
2377   {ARM::RSBSrsr, ARM::RSBrsr},
2378 
2379   {ARM::tADDSi3, ARM::tADDi3},
2380   {ARM::tADDSi8, ARM::tADDi8},
2381   {ARM::tADDSrr, ARM::tADDrr},
2382   {ARM::tADCS, ARM::tADC},
2383 
2384   {ARM::tSUBSi3, ARM::tSUBi3},
2385   {ARM::tSUBSi8, ARM::tSUBi8},
2386   {ARM::tSUBSrr, ARM::tSUBrr},
2387   {ARM::tSBCS, ARM::tSBC},
2388   {ARM::tRSBS, ARM::tRSB},
2389   {ARM::tLSLSri, ARM::tLSLri},
2390 
2391   {ARM::t2ADDSri, ARM::t2ADDri},
2392   {ARM::t2ADDSrr, ARM::t2ADDrr},
2393   {ARM::t2ADDSrs, ARM::t2ADDrs},
2394 
2395   {ARM::t2SUBSri, ARM::t2SUBri},
2396   {ARM::t2SUBSrr, ARM::t2SUBrr},
2397   {ARM::t2SUBSrs, ARM::t2SUBrs},
2398 
2399   {ARM::t2RSBSri, ARM::t2RSBri},
2400   {ARM::t2RSBSrs, ARM::t2RSBrs},
2401 };
2402 
2403 unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2404   for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i)
2405     if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc)
2406       return AddSubFlagsOpcodeMap[i].MachineOpc;
2407   return 0;
2408 }
2409 
2410 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2411                                    MachineBasicBlock::iterator &MBBI,
2412                                    const DebugLoc &dl, Register DestReg,
2413                                    Register BaseReg, int NumBytes,
2414                                    ARMCC::CondCodes Pred, Register PredReg,
2415                                    const ARMBaseInstrInfo &TII,
2416                                    unsigned MIFlags) {
2417   if (NumBytes == 0 && DestReg != BaseReg) {
2418     BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg)
2419         .addReg(BaseReg, RegState::Kill)
2420         .add(predOps(Pred, PredReg))
2421         .add(condCodeOp())
2422         .setMIFlags(MIFlags);
2423     return;
2424   }
2425 
2426   bool isSub = NumBytes < 0;
2427   if (isSub) NumBytes = -NumBytes;
2428 
2429   while (NumBytes) {
2430     unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes);
2431     unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt);
2432     assert(ThisVal && "Didn't extract field correctly");
2433 
2434     // We will handle these bits from offset, clear them.
2435     NumBytes &= ~ThisVal;
2436 
2437     assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2438 
2439     // Build the new ADD / SUB.
2440     unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2441     BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
2442         .addReg(BaseReg, RegState::Kill)
2443         .addImm(ThisVal)
2444         .add(predOps(Pred, PredReg))
2445         .add(condCodeOp())
2446         .setMIFlags(MIFlags);
2447     BaseReg = DestReg;
2448   }
2449 }
2450 
2451 bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2452                                       MachineFunction &MF, MachineInstr *MI,
2453                                       unsigned NumBytes) {
2454   // This optimisation potentially adds lots of load and store
2455   // micro-operations, it's only really a great benefit to code-size.
2456   if (!Subtarget.hasMinSize())
2457     return false;
2458 
2459   // If only one register is pushed/popped, LLVM can use an LDR/STR
2460   // instead. We can't modify those so make sure we're dealing with an
2461   // instruction we understand.
2462   bool IsPop = isPopOpcode(MI->getOpcode());
2463   bool IsPush = isPushOpcode(MI->getOpcode());
2464   if (!IsPush && !IsPop)
2465     return false;
2466 
2467   bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2468                       MI->getOpcode() == ARM::VLDMDIA_UPD;
2469   bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2470                      MI->getOpcode() == ARM::tPOP ||
2471                      MI->getOpcode() == ARM::tPOP_RET;
2472 
2473   assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2474                           MI->getOperand(1).getReg() == ARM::SP)) &&
2475          "trying to fold sp update into non-sp-updating push/pop");
2476 
2477   // The VFP push & pop act on D-registers, so we can only fold an adjustment
2478   // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2479   // if this is violated.
2480   if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2481     return false;
2482 
2483   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2484   // pred) so the list starts at 4. Thumb1 starts after the predicate.
2485   int RegListIdx = IsT1PushPop ? 2 : 4;
2486 
2487   // Calculate the space we'll need in terms of registers.
2488   unsigned RegsNeeded;
2489   const TargetRegisterClass *RegClass;
2490   if (IsVFPPushPop) {
2491     RegsNeeded = NumBytes / 8;
2492     RegClass = &ARM::DPRRegClass;
2493   } else {
2494     RegsNeeded = NumBytes / 4;
2495     RegClass = &ARM::GPRRegClass;
2496   }
2497 
2498   // We're going to have to strip all list operands off before
2499   // re-adding them since the order matters, so save the existing ones
2500   // for later.
2501   SmallVector<MachineOperand, 4> RegList;
2502 
2503   // We're also going to need the first register transferred by this
2504   // instruction, which won't necessarily be the first register in the list.
2505   unsigned FirstRegEnc = -1;
2506 
2507   const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2508   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2509     MachineOperand &MO = MI->getOperand(i);
2510     RegList.push_back(MO);
2511 
2512     if (MO.isReg() && !MO.isImplicit() &&
2513         TRI->getEncodingValue(MO.getReg()) < FirstRegEnc)
2514       FirstRegEnc = TRI->getEncodingValue(MO.getReg());
2515   }
2516 
2517   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
2518 
2519   // Now try to find enough space in the reglist to allocate NumBytes.
2520   for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2521        --CurRegEnc) {
2522     unsigned CurReg = RegClass->getRegister(CurRegEnc);
2523     if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(ARM::R7))
2524       continue;
2525     if (!IsPop) {
2526       // Pushing any register is completely harmless, mark the register involved
2527       // as undef since we don't care about its value and must not restore it
2528       // during stack unwinding.
2529       RegList.push_back(MachineOperand::CreateReg(CurReg, false, false,
2530                                                   false, false, true));
2531       --RegsNeeded;
2532       continue;
2533     }
2534 
2535     // However, we can only pop an extra register if it's not live. For
2536     // registers live within the function we might clobber a return value
2537     // register; the other way a register can be live here is if it's
2538     // callee-saved.
2539     if (isCalleeSavedRegister(CurReg, CSRegs) ||
2540         MI->getParent()->computeRegisterLiveness(TRI, CurReg, MI) !=
2541         MachineBasicBlock::LQR_Dead) {
2542       // VFP pops don't allow holes in the register list, so any skip is fatal
2543       // for our transformation. GPR pops do, so we should just keep looking.
2544       if (IsVFPPushPop)
2545         return false;
2546       else
2547         continue;
2548     }
2549 
2550     // Mark the unimportant registers as <def,dead> in the POP.
2551     RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false,
2552                                                 true));
2553     --RegsNeeded;
2554   }
2555 
2556   if (RegsNeeded > 0)
2557     return false;
2558 
2559   // Finally we know we can profitably perform the optimisation so go
2560   // ahead: strip all existing registers off and add them back again
2561   // in the right order.
2562   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2563     MI->RemoveOperand(i);
2564 
2565   // Add the complete list back in.
2566   MachineInstrBuilder MIB(MF, &*MI);
2567   for (int i = RegList.size() - 1; i >= 0; --i)
2568     MIB.add(RegList[i]);
2569 
2570   return true;
2571 }
2572 
2573 bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2574                                 Register FrameReg, int &Offset,
2575                                 const ARMBaseInstrInfo &TII) {
2576   unsigned Opcode = MI.getOpcode();
2577   const MCInstrDesc &Desc = MI.getDesc();
2578   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2579   bool isSub = false;
2580 
2581   // Memory operands in inline assembly always use AddrMode2.
2582   if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2583     AddrMode = ARMII::AddrMode2;
2584 
2585   if (Opcode == ARM::ADDri) {
2586     Offset += MI.getOperand(FrameRegIdx+1).getImm();
2587     if (Offset == 0) {
2588       // Turn it into a move.
2589       MI.setDesc(TII.get(ARM::MOVr));
2590       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2591       MI.RemoveOperand(FrameRegIdx+1);
2592       Offset = 0;
2593       return true;
2594     } else if (Offset < 0) {
2595       Offset = -Offset;
2596       isSub = true;
2597       MI.setDesc(TII.get(ARM::SUBri));
2598     }
2599 
2600     // Common case: small offset, fits into instruction.
2601     if (ARM_AM::getSOImmVal(Offset) != -1) {
2602       // Replace the FrameIndex with sp / fp
2603       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2604       MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset);
2605       Offset = 0;
2606       return true;
2607     }
2608 
2609     // Otherwise, pull as much of the immedidate into this ADDri/SUBri
2610     // as possible.
2611     unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset);
2612     unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt);
2613 
2614     // We will handle these bits from offset, clear them.
2615     Offset &= ~ThisImmVal;
2616 
2617     // Get the properly encoded SOImmVal field.
2618     assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2619            "Bit extraction didn't work?");
2620     MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal);
2621  } else {
2622     unsigned ImmIdx = 0;
2623     int InstrOffs = 0;
2624     unsigned NumBits = 0;
2625     unsigned Scale = 1;
2626     switch (AddrMode) {
2627     case ARMII::AddrMode_i12:
2628       ImmIdx = FrameRegIdx + 1;
2629       InstrOffs = MI.getOperand(ImmIdx).getImm();
2630       NumBits = 12;
2631       break;
2632     case ARMII::AddrMode2:
2633       ImmIdx = FrameRegIdx+2;
2634       InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm());
2635       if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2636         InstrOffs *= -1;
2637       NumBits = 12;
2638       break;
2639     case ARMII::AddrMode3:
2640       ImmIdx = FrameRegIdx+2;
2641       InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm());
2642       if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2643         InstrOffs *= -1;
2644       NumBits = 8;
2645       break;
2646     case ARMII::AddrMode4:
2647     case ARMII::AddrMode6:
2648       // Can't fold any offset even if it's zero.
2649       return false;
2650     case ARMII::AddrMode5:
2651       ImmIdx = FrameRegIdx+1;
2652       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2653       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2654         InstrOffs *= -1;
2655       NumBits = 8;
2656       Scale = 4;
2657       break;
2658     case ARMII::AddrMode5FP16:
2659       ImmIdx = FrameRegIdx+1;
2660       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2661       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2662         InstrOffs *= -1;
2663       NumBits = 8;
2664       Scale = 2;
2665       break;
2666     case ARMII::AddrModeT2_i7:
2667     case ARMII::AddrModeT2_i7s2:
2668     case ARMII::AddrModeT2_i7s4:
2669       ImmIdx = FrameRegIdx+1;
2670       InstrOffs = MI.getOperand(ImmIdx).getImm();
2671       NumBits = 7;
2672       Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2673                AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2674       break;
2675     default:
2676       llvm_unreachable("Unsupported addressing mode!");
2677     }
2678 
2679     Offset += InstrOffs * Scale;
2680     assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2681     if (Offset < 0) {
2682       Offset = -Offset;
2683       isSub = true;
2684     }
2685 
2686     // Attempt to fold address comp. if opcode has offset bits
2687     if (NumBits > 0) {
2688       // Common case: small offset, fits into instruction.
2689       MachineOperand &ImmOp = MI.getOperand(ImmIdx);
2690       int ImmedOffset = Offset / Scale;
2691       unsigned Mask = (1 << NumBits) - 1;
2692       if ((unsigned)Offset <= Mask * Scale) {
2693         // Replace the FrameIndex with sp
2694         MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2695         // FIXME: When addrmode2 goes away, this will simplify (like the
2696         // T2 version), as the LDR.i12 versions don't need the encoding
2697         // tricks for the offset value.
2698         if (isSub) {
2699           if (AddrMode == ARMII::AddrMode_i12)
2700             ImmedOffset = -ImmedOffset;
2701           else
2702             ImmedOffset |= 1 << NumBits;
2703         }
2704         ImmOp.ChangeToImmediate(ImmedOffset);
2705         Offset = 0;
2706         return true;
2707       }
2708 
2709       // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2710       ImmedOffset = ImmedOffset & Mask;
2711       if (isSub) {
2712         if (AddrMode == ARMII::AddrMode_i12)
2713           ImmedOffset = -ImmedOffset;
2714         else
2715           ImmedOffset |= 1 << NumBits;
2716       }
2717       ImmOp.ChangeToImmediate(ImmedOffset);
2718       Offset &= ~(Mask*Scale);
2719     }
2720   }
2721 
2722   Offset = (isSub) ? -Offset : Offset;
2723   return Offset == 0;
2724 }
2725 
2726 /// analyzeCompare - For a comparison instruction, return the source registers
2727 /// in SrcReg and SrcReg2 if having two register operands, and the value it
2728 /// compares against in CmpValue. Return true if the comparison instruction
2729 /// can be analyzed.
2730 bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2731                                       Register &SrcReg2, int &CmpMask,
2732                                       int &CmpValue) const {
2733   switch (MI.getOpcode()) {
2734   default: break;
2735   case ARM::CMPri:
2736   case ARM::t2CMPri:
2737   case ARM::tCMPi8:
2738     SrcReg = MI.getOperand(0).getReg();
2739     SrcReg2 = 0;
2740     CmpMask = ~0;
2741     CmpValue = MI.getOperand(1).getImm();
2742     return true;
2743   case ARM::CMPrr:
2744   case ARM::t2CMPrr:
2745   case ARM::tCMPr:
2746     SrcReg = MI.getOperand(0).getReg();
2747     SrcReg2 = MI.getOperand(1).getReg();
2748     CmpMask = ~0;
2749     CmpValue = 0;
2750     return true;
2751   case ARM::TSTri:
2752   case ARM::t2TSTri:
2753     SrcReg = MI.getOperand(0).getReg();
2754     SrcReg2 = 0;
2755     CmpMask = MI.getOperand(1).getImm();
2756     CmpValue = 0;
2757     return true;
2758   }
2759 
2760   return false;
2761 }
2762 
2763 /// isSuitableForMask - Identify a suitable 'and' instruction that
2764 /// operates on the given source register and applies the same mask
2765 /// as a 'tst' instruction. Provide a limited look-through for copies.
2766 /// When successful, MI will hold the found instruction.
2767 static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2768                               int CmpMask, bool CommonUse) {
2769   switch (MI->getOpcode()) {
2770     case ARM::ANDri:
2771     case ARM::t2ANDri:
2772       if (CmpMask != MI->getOperand(2).getImm())
2773         return false;
2774       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
2775         return true;
2776       break;
2777   }
2778 
2779   return false;
2780 }
2781 
2782 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2783 /// the condition code if we modify the instructions such that flags are
2784 /// set by ADD(a,b,X).
2785 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2786   switch (CC) {
2787   default: return ARMCC::AL;
2788   case ARMCC::HS: return ARMCC::LO;
2789   case ARMCC::LO: return ARMCC::HS;
2790   case ARMCC::VS: return ARMCC::VS;
2791   case ARMCC::VC: return ARMCC::VC;
2792   }
2793 }
2794 
2795 /// isRedundantFlagInstr - check whether the first instruction, whose only
2796 /// purpose is to update flags, can be made redundant.
2797 /// CMPrr can be made redundant by SUBrr if the operands are the same.
2798 /// CMPri can be made redundant by SUBri if the operands are the same.
2799 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2800 /// This function can be extended later on.
2801 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2802                                         Register SrcReg, Register SrcReg2,
2803                                         int ImmValue, const MachineInstr *OI,
2804                                         bool &IsThumb1) {
2805   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2806       (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2807       ((OI->getOperand(1).getReg() == SrcReg &&
2808         OI->getOperand(2).getReg() == SrcReg2) ||
2809        (OI->getOperand(1).getReg() == SrcReg2 &&
2810         OI->getOperand(2).getReg() == SrcReg))) {
2811     IsThumb1 = false;
2812     return true;
2813   }
2814 
2815   if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2816       ((OI->getOperand(2).getReg() == SrcReg &&
2817         OI->getOperand(3).getReg() == SrcReg2) ||
2818        (OI->getOperand(2).getReg() == SrcReg2 &&
2819         OI->getOperand(3).getReg() == SrcReg))) {
2820     IsThumb1 = true;
2821     return true;
2822   }
2823 
2824   if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2825       (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2826       OI->getOperand(1).getReg() == SrcReg &&
2827       OI->getOperand(2).getImm() == ImmValue) {
2828     IsThumb1 = false;
2829     return true;
2830   }
2831 
2832   if (CmpI->getOpcode() == ARM::tCMPi8 &&
2833       (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2834       OI->getOperand(2).getReg() == SrcReg &&
2835       OI->getOperand(3).getImm() == ImmValue) {
2836     IsThumb1 = true;
2837     return true;
2838   }
2839 
2840   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2841       (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2842        OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2843       OI->getOperand(0).isReg() && OI->getOperand(1).isReg() &&
2844       OI->getOperand(0).getReg() == SrcReg &&
2845       OI->getOperand(1).getReg() == SrcReg2) {
2846     IsThumb1 = false;
2847     return true;
2848   }
2849 
2850   if (CmpI->getOpcode() == ARM::tCMPr &&
2851       (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2852        OI->getOpcode() == ARM::tADDrr) &&
2853       OI->getOperand(0).getReg() == SrcReg &&
2854       OI->getOperand(2).getReg() == SrcReg2) {
2855     IsThumb1 = true;
2856     return true;
2857   }
2858 
2859   return false;
2860 }
2861 
2862 static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2863   switch (MI->getOpcode()) {
2864   default: return false;
2865   case ARM::tLSLri:
2866   case ARM::tLSRri:
2867   case ARM::tLSLrr:
2868   case ARM::tLSRrr:
2869   case ARM::tSUBrr:
2870   case ARM::tADDrr:
2871   case ARM::tADDi3:
2872   case ARM::tADDi8:
2873   case ARM::tSUBi3:
2874   case ARM::tSUBi8:
2875   case ARM::tMUL:
2876   case ARM::tADC:
2877   case ARM::tSBC:
2878   case ARM::tRSB:
2879   case ARM::tAND:
2880   case ARM::tORR:
2881   case ARM::tEOR:
2882   case ARM::tBIC:
2883   case ARM::tMVN:
2884   case ARM::tASRri:
2885   case ARM::tASRrr:
2886   case ARM::tROR:
2887     IsThumb1 = true;
2888     LLVM_FALLTHROUGH;
2889   case ARM::RSBrr:
2890   case ARM::RSBri:
2891   case ARM::RSCrr:
2892   case ARM::RSCri:
2893   case ARM::ADDrr:
2894   case ARM::ADDri:
2895   case ARM::ADCrr:
2896   case ARM::ADCri:
2897   case ARM::SUBrr:
2898   case ARM::SUBri:
2899   case ARM::SBCrr:
2900   case ARM::SBCri:
2901   case ARM::t2RSBri:
2902   case ARM::t2ADDrr:
2903   case ARM::t2ADDri:
2904   case ARM::t2ADCrr:
2905   case ARM::t2ADCri:
2906   case ARM::t2SUBrr:
2907   case ARM::t2SUBri:
2908   case ARM::t2SBCrr:
2909   case ARM::t2SBCri:
2910   case ARM::ANDrr:
2911   case ARM::ANDri:
2912   case ARM::t2ANDrr:
2913   case ARM::t2ANDri:
2914   case ARM::ORRrr:
2915   case ARM::ORRri:
2916   case ARM::t2ORRrr:
2917   case ARM::t2ORRri:
2918   case ARM::EORrr:
2919   case ARM::EORri:
2920   case ARM::t2EORrr:
2921   case ARM::t2EORri:
2922   case ARM::t2LSRri:
2923   case ARM::t2LSRrr:
2924   case ARM::t2LSLri:
2925   case ARM::t2LSLrr:
2926     return true;
2927   }
2928 }
2929 
2930 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
2931 /// comparison into one that sets the zero bit in the flags register;
2932 /// Remove a redundant Compare instruction if an earlier instruction can set the
2933 /// flags in the same way as Compare.
2934 /// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2935 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2936 /// condition code of instructions which use the flags.
2937 bool ARMBaseInstrInfo::optimizeCompareInstr(
2938     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
2939     int CmpValue, const MachineRegisterInfo *MRI) const {
2940   // Get the unique definition of SrcReg.
2941   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2942   if (!MI) return false;
2943 
2944   // Masked compares sometimes use the same register as the corresponding 'and'.
2945   if (CmpMask != ~0) {
2946     if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(*MI)) {
2947       MI = nullptr;
2948       for (MachineRegisterInfo::use_instr_iterator
2949            UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end();
2950            UI != UE; ++UI) {
2951         if (UI->getParent() != CmpInstr.getParent())
2952           continue;
2953         MachineInstr *PotentialAND = &*UI;
2954         if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
2955             isPredicated(*PotentialAND))
2956           continue;
2957         MI = PotentialAND;
2958         break;
2959       }
2960       if (!MI) return false;
2961     }
2962   }
2963 
2964   // Get ready to iterate backward from CmpInstr.
2965   MachineBasicBlock::iterator I = CmpInstr, E = MI,
2966                               B = CmpInstr.getParent()->begin();
2967 
2968   // Early exit if CmpInstr is at the beginning of the BB.
2969   if (I == B) return false;
2970 
2971   // There are two possible candidates which can be changed to set CPSR:
2972   // One is MI, the other is a SUB or ADD instruction.
2973   // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
2974   // ADDr[ri](r1, r2, X).
2975   // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
2976   MachineInstr *SubAdd = nullptr;
2977   if (SrcReg2 != 0)
2978     // MI is not a candidate for CMPrr.
2979     MI = nullptr;
2980   else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
2981     // Conservatively refuse to convert an instruction which isn't in the same
2982     // BB as the comparison.
2983     // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
2984     // Thus we cannot return here.
2985     if (CmpInstr.getOpcode() == ARM::CMPri ||
2986         CmpInstr.getOpcode() == ARM::t2CMPri ||
2987         CmpInstr.getOpcode() == ARM::tCMPi8)
2988       MI = nullptr;
2989     else
2990       return false;
2991   }
2992 
2993   bool IsThumb1 = false;
2994   if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
2995     return false;
2996 
2997   // We also want to do this peephole for cases like this: if (a*b == 0),
2998   // and optimise away the CMP instruction from the generated code sequence:
2999   // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
3000   // resulting from the select instruction, but these MOVS instructions for
3001   // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
3002   // However, if we only have MOVS instructions in between the CMP and the
3003   // other instruction (the MULS in this example), then the CPSR is dead so we
3004   // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
3005   // reordering and then continue the analysis hoping we can eliminate the
3006   // CMP. This peephole works on the vregs, so is still in SSA form. As a
3007   // consequence, the movs won't redefine/kill the MUL operands which would
3008   // make this reordering illegal.
3009   const TargetRegisterInfo *TRI = &getRegisterInfo();
3010   if (MI && IsThumb1) {
3011     --I;
3012     if (I != E && !MI->readsRegister(ARM::CPSR, TRI)) {
3013       bool CanReorder = true;
3014       for (; I != E; --I) {
3015         if (I->getOpcode() != ARM::tMOVi8) {
3016           CanReorder = false;
3017           break;
3018         }
3019       }
3020       if (CanReorder) {
3021         MI = MI->removeFromParent();
3022         E = CmpInstr;
3023         CmpInstr.getParent()->insert(E, MI);
3024       }
3025     }
3026     I = CmpInstr;
3027     E = MI;
3028   }
3029 
3030   // Check that CPSR isn't set between the comparison instruction and the one we
3031   // want to change. At the same time, search for SubAdd.
3032   bool SubAddIsThumb1 = false;
3033   do {
3034     const MachineInstr &Instr = *--I;
3035 
3036     // Check whether CmpInstr can be made redundant by the current instruction.
3037     if (isRedundantFlagInstr(&CmpInstr, SrcReg, SrcReg2, CmpValue, &Instr,
3038                              SubAddIsThumb1)) {
3039       SubAdd = &*I;
3040       break;
3041     }
3042 
3043     // Allow E (which was initially MI) to be SubAdd but do not search before E.
3044     if (I == E)
3045       break;
3046 
3047     if (Instr.modifiesRegister(ARM::CPSR, TRI) ||
3048         Instr.readsRegister(ARM::CPSR, TRI))
3049       // This instruction modifies or uses CPSR after the one we want to
3050       // change. We can't do this transformation.
3051       return false;
3052 
3053     if (I == B) {
3054       // In some cases, we scan the use-list of an instruction for an AND;
3055       // that AND is in the same BB, but may not be scheduled before the
3056       // corresponding TST.  In that case, bail out.
3057       //
3058       // FIXME: We could try to reschedule the AND.
3059       return false;
3060     }
3061   } while (true);
3062 
3063   // Return false if no candidates exist.
3064   if (!MI && !SubAdd)
3065     return false;
3066 
3067   // If we found a SubAdd, use it as it will be closer to the CMP
3068   if (SubAdd) {
3069     MI = SubAdd;
3070     IsThumb1 = SubAddIsThumb1;
3071   }
3072 
3073   // We can't use a predicated instruction - it doesn't always write the flags.
3074   if (isPredicated(*MI))
3075     return false;
3076 
3077   // Scan forward for the use of CPSR
3078   // When checking against MI: if it's a conditional code that requires
3079   // checking of the V bit or C bit, then this is not safe to do.
3080   // It is safe to remove CmpInstr if CPSR is redefined or killed.
3081   // If we are done with the basic block, we need to check whether CPSR is
3082   // live-out.
3083   SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
3084       OperandsToUpdate;
3085   bool isSafe = false;
3086   I = CmpInstr;
3087   E = CmpInstr.getParent()->end();
3088   while (!isSafe && ++I != E) {
3089     const MachineInstr &Instr = *I;
3090     for (unsigned IO = 0, EO = Instr.getNumOperands();
3091          !isSafe && IO != EO; ++IO) {
3092       const MachineOperand &MO = Instr.getOperand(IO);
3093       if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) {
3094         isSafe = true;
3095         break;
3096       }
3097       if (!MO.isReg() || MO.getReg() != ARM::CPSR)
3098         continue;
3099       if (MO.isDef()) {
3100         isSafe = true;
3101         break;
3102       }
3103       // Condition code is after the operand before CPSR except for VSELs.
3104       ARMCC::CondCodes CC;
3105       bool IsInstrVSel = true;
3106       switch (Instr.getOpcode()) {
3107       default:
3108         IsInstrVSel = false;
3109         CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm();
3110         break;
3111       case ARM::VSELEQD:
3112       case ARM::VSELEQS:
3113       case ARM::VSELEQH:
3114         CC = ARMCC::EQ;
3115         break;
3116       case ARM::VSELGTD:
3117       case ARM::VSELGTS:
3118       case ARM::VSELGTH:
3119         CC = ARMCC::GT;
3120         break;
3121       case ARM::VSELGED:
3122       case ARM::VSELGES:
3123       case ARM::VSELGEH:
3124         CC = ARMCC::GE;
3125         break;
3126       case ARM::VSELVSD:
3127       case ARM::VSELVSS:
3128       case ARM::VSELVSH:
3129         CC = ARMCC::VS;
3130         break;
3131       }
3132 
3133       if (SubAdd) {
3134         // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3135         // on CMP needs to be updated to be based on SUB.
3136         // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3137         // needs to be modified.
3138         // Push the condition code operands to OperandsToUpdate.
3139         // If it is safe to remove CmpInstr, the condition code of these
3140         // operands will be modified.
3141         unsigned Opc = SubAdd->getOpcode();
3142         bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3143                      Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3144                      Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3145                      Opc == ARM::tSUBi8;
3146         unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3147         if (!IsSub ||
3148             (SrcReg2 != 0 && SubAdd->getOperand(OpI).getReg() == SrcReg2 &&
3149              SubAdd->getOperand(OpI + 1).getReg() == SrcReg)) {
3150           // VSel doesn't support condition code update.
3151           if (IsInstrVSel)
3152             return false;
3153           // Ensure we can swap the condition.
3154           ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3155           if (NewCC == ARMCC::AL)
3156             return false;
3157           OperandsToUpdate.push_back(
3158               std::make_pair(&((*I).getOperand(IO - 1)), NewCC));
3159         }
3160       } else {
3161         // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3162         switch (CC) {
3163         case ARMCC::EQ: // Z
3164         case ARMCC::NE: // Z
3165         case ARMCC::MI: // N
3166         case ARMCC::PL: // N
3167         case ARMCC::AL: // none
3168           // CPSR can be used multiple times, we should continue.
3169           break;
3170         case ARMCC::HS: // C
3171         case ARMCC::LO: // C
3172         case ARMCC::VS: // V
3173         case ARMCC::VC: // V
3174         case ARMCC::HI: // C Z
3175         case ARMCC::LS: // C Z
3176         case ARMCC::GE: // N V
3177         case ARMCC::LT: // N V
3178         case ARMCC::GT: // Z N V
3179         case ARMCC::LE: // Z N V
3180           // The instruction uses the V bit or C bit which is not safe.
3181           return false;
3182         }
3183       }
3184     }
3185   }
3186 
3187   // If CPSR is not killed nor re-defined, we should check whether it is
3188   // live-out. If it is live-out, do not optimize.
3189   if (!isSafe) {
3190     MachineBasicBlock *MBB = CmpInstr.getParent();
3191     for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
3192              SE = MBB->succ_end(); SI != SE; ++SI)
3193       if ((*SI)->isLiveIn(ARM::CPSR))
3194         return false;
3195   }
3196 
3197   // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3198   // set CPSR so this is represented as an explicit output)
3199   if (!IsThumb1) {
3200     MI->getOperand(5).setReg(ARM::CPSR);
3201     MI->getOperand(5).setIsDef(true);
3202   }
3203   assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3204   CmpInstr.eraseFromParent();
3205 
3206   // Modify the condition code of operands in OperandsToUpdate.
3207   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3208   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3209   for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
3210     OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
3211 
3212   MI->clearRegisterDeads(ARM::CPSR);
3213 
3214   return true;
3215 }
3216 
3217 bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3218   // Do not sink MI if it might be used to optimize a redundant compare.
3219   // We heuristically only look at the instruction immediately following MI to
3220   // avoid potentially searching the entire basic block.
3221   if (isPredicated(MI))
3222     return true;
3223   MachineBasicBlock::const_iterator Next = &MI;
3224   ++Next;
3225   Register SrcReg, SrcReg2;
3226   int CmpMask, CmpValue;
3227   bool IsThumb1;
3228   if (Next != MI.getParent()->end() &&
3229       analyzeCompare(*Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3230       isRedundantFlagInstr(&*Next, SrcReg, SrcReg2, CmpValue, &MI, IsThumb1))
3231     return false;
3232   return true;
3233 }
3234 
3235 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3236                                      Register Reg,
3237                                      MachineRegisterInfo *MRI) const {
3238   // Fold large immediates into add, sub, or, xor.
3239   unsigned DefOpc = DefMI.getOpcode();
3240   if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm)
3241     return false;
3242   if (!DefMI.getOperand(1).isImm())
3243     // Could be t2MOVi32imm @xx
3244     return false;
3245 
3246   if (!MRI->hasOneNonDBGUse(Reg))
3247     return false;
3248 
3249   const MCInstrDesc &DefMCID = DefMI.getDesc();
3250   if (DefMCID.hasOptionalDef()) {
3251     unsigned NumOps = DefMCID.getNumOperands();
3252     const MachineOperand &MO = DefMI.getOperand(NumOps - 1);
3253     if (MO.getReg() == ARM::CPSR && !MO.isDead())
3254       // If DefMI defines CPSR and it is not dead, it's obviously not safe
3255       // to delete DefMI.
3256       return false;
3257   }
3258 
3259   const MCInstrDesc &UseMCID = UseMI.getDesc();
3260   if (UseMCID.hasOptionalDef()) {
3261     unsigned NumOps = UseMCID.getNumOperands();
3262     if (UseMI.getOperand(NumOps - 1).getReg() == ARM::CPSR)
3263       // If the instruction sets the flag, do not attempt this optimization
3264       // since it may change the semantics of the code.
3265       return false;
3266   }
3267 
3268   unsigned UseOpc = UseMI.getOpcode();
3269   unsigned NewUseOpc = 0;
3270   uint32_t ImmVal = (uint32_t)DefMI.getOperand(1).getImm();
3271   uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3272   bool Commute = false;
3273   switch (UseOpc) {
3274   default: return false;
3275   case ARM::SUBrr:
3276   case ARM::ADDrr:
3277   case ARM::ORRrr:
3278   case ARM::EORrr:
3279   case ARM::t2SUBrr:
3280   case ARM::t2ADDrr:
3281   case ARM::t2ORRrr:
3282   case ARM::t2EORrr: {
3283     Commute = UseMI.getOperand(2).getReg() != Reg;
3284     switch (UseOpc) {
3285     default: break;
3286     case ARM::ADDrr:
3287     case ARM::SUBrr:
3288       if (UseOpc == ARM::SUBrr && Commute)
3289         return false;
3290 
3291       // ADD/SUB are special because they're essentially the same operation, so
3292       // we can handle a larger range of immediates.
3293       if (ARM_AM::isSOImmTwoPartVal(ImmVal))
3294         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3295       else if (ARM_AM::isSOImmTwoPartVal(-ImmVal)) {
3296         ImmVal = -ImmVal;
3297         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3298       } else
3299         return false;
3300       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3301       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3302       break;
3303     case ARM::ORRrr:
3304     case ARM::EORrr:
3305       if (!ARM_AM::isSOImmTwoPartVal(ImmVal))
3306         return false;
3307       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3308       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3309       switch (UseOpc) {
3310       default: break;
3311       case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3312       case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3313       }
3314       break;
3315     case ARM::t2ADDrr:
3316     case ARM::t2SUBrr: {
3317       if (UseOpc == ARM::t2SUBrr && Commute)
3318         return false;
3319 
3320       // ADD/SUB are special because they're essentially the same operation, so
3321       // we can handle a larger range of immediates.
3322       const bool ToSP = DefMI.getOperand(0).getReg() == ARM::SP;
3323       const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3324       const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3325       if (ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3326         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3327       else if (ARM_AM::isT2SOImmTwoPartVal(-ImmVal)) {
3328         ImmVal = -ImmVal;
3329         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3330       } else
3331         return false;
3332       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3333       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3334       break;
3335     }
3336     case ARM::t2ORRrr:
3337     case ARM::t2EORrr:
3338       if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3339         return false;
3340       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3341       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3342       switch (UseOpc) {
3343       default: break;
3344       case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3345       case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3346       }
3347       break;
3348     }
3349   }
3350   }
3351 
3352   unsigned OpIdx = Commute ? 2 : 1;
3353   Register Reg1 = UseMI.getOperand(OpIdx).getReg();
3354   bool isKill = UseMI.getOperand(OpIdx).isKill();
3355   const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3356   Register NewReg = MRI->createVirtualRegister(TRC);
3357   BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), get(NewUseOpc),
3358           NewReg)
3359       .addReg(Reg1, getKillRegState(isKill))
3360       .addImm(SOImmValV1)
3361       .add(predOps(ARMCC::AL))
3362       .add(condCodeOp());
3363   UseMI.setDesc(get(NewUseOpc));
3364   UseMI.getOperand(1).setReg(NewReg);
3365   UseMI.getOperand(1).setIsKill();
3366   UseMI.getOperand(2).ChangeToImmediate(SOImmValV2);
3367   DefMI.eraseFromParent();
3368   // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3369   // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3370   // Then the below code will not be needed, as the input/output register
3371   // classes will be rgpr or gprSP.
3372   // For now, we fix the UseMI operand explicitly here:
3373   switch(NewUseOpc){
3374     case ARM::t2ADDspImm:
3375     case ARM::t2SUBspImm:
3376     case ARM::t2ADDri:
3377     case ARM::t2SUBri:
3378       MRI->setRegClass(UseMI.getOperand(0).getReg(), TRC);
3379   }
3380   return true;
3381 }
3382 
3383 static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3384                                         const MachineInstr &MI) {
3385   switch (MI.getOpcode()) {
3386   default: {
3387     const MCInstrDesc &Desc = MI.getDesc();
3388     int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
3389     assert(UOps >= 0 && "bad # UOps");
3390     return UOps;
3391   }
3392 
3393   case ARM::LDRrs:
3394   case ARM::LDRBrs:
3395   case ARM::STRrs:
3396   case ARM::STRBrs: {
3397     unsigned ShOpVal = MI.getOperand(3).getImm();
3398     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3399     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3400     if (!isSub &&
3401         (ShImm == 0 ||
3402          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3403           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3404       return 1;
3405     return 2;
3406   }
3407 
3408   case ARM::LDRH:
3409   case ARM::STRH: {
3410     if (!MI.getOperand(2).getReg())
3411       return 1;
3412 
3413     unsigned ShOpVal = MI.getOperand(3).getImm();
3414     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3415     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3416     if (!isSub &&
3417         (ShImm == 0 ||
3418          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3419           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3420       return 1;
3421     return 2;
3422   }
3423 
3424   case ARM::LDRSB:
3425   case ARM::LDRSH:
3426     return (ARM_AM::getAM3Op(MI.getOperand(3).getImm()) == ARM_AM::sub) ? 3 : 2;
3427 
3428   case ARM::LDRSB_POST:
3429   case ARM::LDRSH_POST: {
3430     Register Rt = MI.getOperand(0).getReg();
3431     Register Rm = MI.getOperand(3).getReg();
3432     return (Rt == Rm) ? 4 : 3;
3433   }
3434 
3435   case ARM::LDR_PRE_REG:
3436   case ARM::LDRB_PRE_REG: {
3437     Register Rt = MI.getOperand(0).getReg();
3438     Register Rm = MI.getOperand(3).getReg();
3439     if (Rt == Rm)
3440       return 3;
3441     unsigned ShOpVal = MI.getOperand(4).getImm();
3442     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3443     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3444     if (!isSub &&
3445         (ShImm == 0 ||
3446          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3447           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3448       return 2;
3449     return 3;
3450   }
3451 
3452   case ARM::STR_PRE_REG:
3453   case ARM::STRB_PRE_REG: {
3454     unsigned ShOpVal = MI.getOperand(4).getImm();
3455     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3456     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3457     if (!isSub &&
3458         (ShImm == 0 ||
3459          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3460           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3461       return 2;
3462     return 3;
3463   }
3464 
3465   case ARM::LDRH_PRE:
3466   case ARM::STRH_PRE: {
3467     Register Rt = MI.getOperand(0).getReg();
3468     Register Rm = MI.getOperand(3).getReg();
3469     if (!Rm)
3470       return 2;
3471     if (Rt == Rm)
3472       return 3;
3473     return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 3 : 2;
3474   }
3475 
3476   case ARM::LDR_POST_REG:
3477   case ARM::LDRB_POST_REG:
3478   case ARM::LDRH_POST: {
3479     Register Rt = MI.getOperand(0).getReg();
3480     Register Rm = MI.getOperand(3).getReg();
3481     return (Rt == Rm) ? 3 : 2;
3482   }
3483 
3484   case ARM::LDR_PRE_IMM:
3485   case ARM::LDRB_PRE_IMM:
3486   case ARM::LDR_POST_IMM:
3487   case ARM::LDRB_POST_IMM:
3488   case ARM::STRB_POST_IMM:
3489   case ARM::STRB_POST_REG:
3490   case ARM::STRB_PRE_IMM:
3491   case ARM::STRH_POST:
3492   case ARM::STR_POST_IMM:
3493   case ARM::STR_POST_REG:
3494   case ARM::STR_PRE_IMM:
3495     return 2;
3496 
3497   case ARM::LDRSB_PRE:
3498   case ARM::LDRSH_PRE: {
3499     Register Rm = MI.getOperand(3).getReg();
3500     if (Rm == 0)
3501       return 3;
3502     Register Rt = MI.getOperand(0).getReg();
3503     if (Rt == Rm)
3504       return 4;
3505     unsigned ShOpVal = MI.getOperand(4).getImm();
3506     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3507     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3508     if (!isSub &&
3509         (ShImm == 0 ||
3510          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3511           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3512       return 3;
3513     return 4;
3514   }
3515 
3516   case ARM::LDRD: {
3517     Register Rt = MI.getOperand(0).getReg();
3518     Register Rn = MI.getOperand(2).getReg();
3519     Register Rm = MI.getOperand(3).getReg();
3520     if (Rm)
3521       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3522                                                                           : 3;
3523     return (Rt == Rn) ? 3 : 2;
3524   }
3525 
3526   case ARM::STRD: {
3527     Register Rm = MI.getOperand(3).getReg();
3528     if (Rm)
3529       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3530                                                                           : 3;
3531     return 2;
3532   }
3533 
3534   case ARM::LDRD_POST:
3535   case ARM::t2LDRD_POST:
3536     return 3;
3537 
3538   case ARM::STRD_POST:
3539   case ARM::t2STRD_POST:
3540     return 4;
3541 
3542   case ARM::LDRD_PRE: {
3543     Register Rt = MI.getOperand(0).getReg();
3544     Register Rn = MI.getOperand(3).getReg();
3545     Register Rm = MI.getOperand(4).getReg();
3546     if (Rm)
3547       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3548                                                                           : 4;
3549     return (Rt == Rn) ? 4 : 3;
3550   }
3551 
3552   case ARM::t2LDRD_PRE: {
3553     Register Rt = MI.getOperand(0).getReg();
3554     Register Rn = MI.getOperand(3).getReg();
3555     return (Rt == Rn) ? 4 : 3;
3556   }
3557 
3558   case ARM::STRD_PRE: {
3559     Register Rm = MI.getOperand(4).getReg();
3560     if (Rm)
3561       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3562                                                                           : 4;
3563     return 3;
3564   }
3565 
3566   case ARM::t2STRD_PRE:
3567     return 3;
3568 
3569   case ARM::t2LDR_POST:
3570   case ARM::t2LDRB_POST:
3571   case ARM::t2LDRB_PRE:
3572   case ARM::t2LDRSBi12:
3573   case ARM::t2LDRSBi8:
3574   case ARM::t2LDRSBpci:
3575   case ARM::t2LDRSBs:
3576   case ARM::t2LDRH_POST:
3577   case ARM::t2LDRH_PRE:
3578   case ARM::t2LDRSBT:
3579   case ARM::t2LDRSB_POST:
3580   case ARM::t2LDRSB_PRE:
3581   case ARM::t2LDRSH_POST:
3582   case ARM::t2LDRSH_PRE:
3583   case ARM::t2LDRSHi12:
3584   case ARM::t2LDRSHi8:
3585   case ARM::t2LDRSHpci:
3586   case ARM::t2LDRSHs:
3587     return 2;
3588 
3589   case ARM::t2LDRDi8: {
3590     Register Rt = MI.getOperand(0).getReg();
3591     Register Rn = MI.getOperand(2).getReg();
3592     return (Rt == Rn) ? 3 : 2;
3593   }
3594 
3595   case ARM::t2STRB_POST:
3596   case ARM::t2STRB_PRE:
3597   case ARM::t2STRBs:
3598   case ARM::t2STRDi8:
3599   case ARM::t2STRH_POST:
3600   case ARM::t2STRH_PRE:
3601   case ARM::t2STRHs:
3602   case ARM::t2STR_POST:
3603   case ARM::t2STR_PRE:
3604   case ARM::t2STRs:
3605     return 2;
3606   }
3607 }
3608 
3609 // Return the number of 32-bit words loaded by LDM or stored by STM. If this
3610 // can't be easily determined return 0 (missing MachineMemOperand).
3611 //
3612 // FIXME: The current MachineInstr design does not support relying on machine
3613 // mem operands to determine the width of a memory access. Instead, we expect
3614 // the target to provide this information based on the instruction opcode and
3615 // operands. However, using MachineMemOperand is the best solution now for
3616 // two reasons:
3617 //
3618 // 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3619 // operands. This is much more dangerous than using the MachineMemOperand
3620 // sizes because CodeGen passes can insert/remove optional machine operands. In
3621 // fact, it's totally incorrect for preRA passes and appears to be wrong for
3622 // postRA passes as well.
3623 //
3624 // 2) getNumLDMAddresses is only used by the scheduling machine model and any
3625 // machine model that calls this should handle the unknown (zero size) case.
3626 //
3627 // Long term, we should require a target hook that verifies MachineMemOperand
3628 // sizes during MC lowering. That target hook should be local to MC lowering
3629 // because we can't ensure that it is aware of other MI forms. Doing this will
3630 // ensure that MachineMemOperands are correctly propagated through all passes.
3631 unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3632   unsigned Size = 0;
3633   for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3634                                   E = MI.memoperands_end();
3635        I != E; ++I) {
3636     Size += (*I)->getSize();
3637   }
3638   // FIXME: The scheduler currently can't handle values larger than 16. But
3639   // the values can actually go up to 32 for floating-point load/store
3640   // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3641   // operations isn't right; we could end up with "extra" memory operands for
3642   // various reasons, like tail merge merging two memory operations.
3643   return std::min(Size / 4, 16U);
3644 }
3645 
3646 static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3647                                                     unsigned NumRegs) {
3648   unsigned UOps = 1 + NumRegs; // 1 for address computation.
3649   switch (Opc) {
3650   default:
3651     break;
3652   case ARM::VLDMDIA_UPD:
3653   case ARM::VLDMDDB_UPD:
3654   case ARM::VLDMSIA_UPD:
3655   case ARM::VLDMSDB_UPD:
3656   case ARM::VSTMDIA_UPD:
3657   case ARM::VSTMDDB_UPD:
3658   case ARM::VSTMSIA_UPD:
3659   case ARM::VSTMSDB_UPD:
3660   case ARM::LDMIA_UPD:
3661   case ARM::LDMDA_UPD:
3662   case ARM::LDMDB_UPD:
3663   case ARM::LDMIB_UPD:
3664   case ARM::STMIA_UPD:
3665   case ARM::STMDA_UPD:
3666   case ARM::STMDB_UPD:
3667   case ARM::STMIB_UPD:
3668   case ARM::tLDMIA_UPD:
3669   case ARM::tSTMIA_UPD:
3670   case ARM::t2LDMIA_UPD:
3671   case ARM::t2LDMDB_UPD:
3672   case ARM::t2STMIA_UPD:
3673   case ARM::t2STMDB_UPD:
3674     ++UOps; // One for base register writeback.
3675     break;
3676   case ARM::LDMIA_RET:
3677   case ARM::tPOP_RET:
3678   case ARM::t2LDMIA_RET:
3679     UOps += 2; // One for base reg wb, one for write to pc.
3680     break;
3681   }
3682   return UOps;
3683 }
3684 
3685 unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3686                                           const MachineInstr &MI) const {
3687   if (!ItinData || ItinData->isEmpty())
3688     return 1;
3689 
3690   const MCInstrDesc &Desc = MI.getDesc();
3691   unsigned Class = Desc.getSchedClass();
3692   int ItinUOps = ItinData->getNumMicroOps(Class);
3693   if (ItinUOps >= 0) {
3694     if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3695       return getNumMicroOpsSwiftLdSt(ItinData, MI);
3696 
3697     return ItinUOps;
3698   }
3699 
3700   unsigned Opc = MI.getOpcode();
3701   switch (Opc) {
3702   default:
3703     llvm_unreachable("Unexpected multi-uops instruction!");
3704   case ARM::VLDMQIA:
3705   case ARM::VSTMQIA:
3706     return 2;
3707 
3708   // The number of uOps for load / store multiple are determined by the number
3709   // registers.
3710   //
3711   // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3712   // same cycle. The scheduling for the first load / store must be done
3713   // separately by assuming the address is not 64-bit aligned.
3714   //
3715   // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3716   // is not 64-bit aligned, then AGU would take an extra cycle.  For VFP / NEON
3717   // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3718   case ARM::VLDMDIA:
3719   case ARM::VLDMDIA_UPD:
3720   case ARM::VLDMDDB_UPD:
3721   case ARM::VLDMSIA:
3722   case ARM::VLDMSIA_UPD:
3723   case ARM::VLDMSDB_UPD:
3724   case ARM::VSTMDIA:
3725   case ARM::VSTMDIA_UPD:
3726   case ARM::VSTMDDB_UPD:
3727   case ARM::VSTMSIA:
3728   case ARM::VSTMSIA_UPD:
3729   case ARM::VSTMSDB_UPD: {
3730     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3731     return (NumRegs / 2) + (NumRegs % 2) + 1;
3732   }
3733 
3734   case ARM::LDMIA_RET:
3735   case ARM::LDMIA:
3736   case ARM::LDMDA:
3737   case ARM::LDMDB:
3738   case ARM::LDMIB:
3739   case ARM::LDMIA_UPD:
3740   case ARM::LDMDA_UPD:
3741   case ARM::LDMDB_UPD:
3742   case ARM::LDMIB_UPD:
3743   case ARM::STMIA:
3744   case ARM::STMDA:
3745   case ARM::STMDB:
3746   case ARM::STMIB:
3747   case ARM::STMIA_UPD:
3748   case ARM::STMDA_UPD:
3749   case ARM::STMDB_UPD:
3750   case ARM::STMIB_UPD:
3751   case ARM::tLDMIA:
3752   case ARM::tLDMIA_UPD:
3753   case ARM::tSTMIA_UPD:
3754   case ARM::tPOP_RET:
3755   case ARM::tPOP:
3756   case ARM::tPUSH:
3757   case ARM::t2LDMIA_RET:
3758   case ARM::t2LDMIA:
3759   case ARM::t2LDMDB:
3760   case ARM::t2LDMIA_UPD:
3761   case ARM::t2LDMDB_UPD:
3762   case ARM::t2STMIA:
3763   case ARM::t2STMDB:
3764   case ARM::t2STMIA_UPD:
3765   case ARM::t2STMDB_UPD: {
3766     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3767     switch (Subtarget.getLdStMultipleTiming()) {
3768     case ARMSubtarget::SingleIssuePlusExtras:
3769       return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3770     case ARMSubtarget::SingleIssue:
3771       // Assume the worst.
3772       return NumRegs;
3773     case ARMSubtarget::DoubleIssue: {
3774       if (NumRegs < 4)
3775         return 2;
3776       // 4 registers would be issued: 2, 2.
3777       // 5 registers would be issued: 2, 2, 1.
3778       unsigned UOps = (NumRegs / 2);
3779       if (NumRegs % 2)
3780         ++UOps;
3781       return UOps;
3782     }
3783     case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3784       unsigned UOps = (NumRegs / 2);
3785       // If there are odd number of registers or if it's not 64-bit aligned,
3786       // then it takes an extra AGU (Address Generation Unit) cycle.
3787       if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3788           (*MI.memoperands_begin())->getAlign() < Align(8))
3789         ++UOps;
3790       return UOps;
3791       }
3792     }
3793   }
3794   }
3795   llvm_unreachable("Didn't find the number of microops");
3796 }
3797 
3798 int
3799 ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3800                                   const MCInstrDesc &DefMCID,
3801                                   unsigned DefClass,
3802                                   unsigned DefIdx, unsigned DefAlign) const {
3803   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3804   if (RegNo <= 0)
3805     // Def is the address writeback.
3806     return ItinData->getOperandCycle(DefClass, DefIdx);
3807 
3808   int DefCycle;
3809   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3810     // (regno / 2) + (regno % 2) + 1
3811     DefCycle = RegNo / 2 + 1;
3812     if (RegNo % 2)
3813       ++DefCycle;
3814   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3815     DefCycle = RegNo;
3816     bool isSLoad = false;
3817 
3818     switch (DefMCID.getOpcode()) {
3819     default: break;
3820     case ARM::VLDMSIA:
3821     case ARM::VLDMSIA_UPD:
3822     case ARM::VLDMSDB_UPD:
3823       isSLoad = true;
3824       break;
3825     }
3826 
3827     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3828     // then it takes an extra cycle.
3829     if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3830       ++DefCycle;
3831   } else {
3832     // Assume the worst.
3833     DefCycle = RegNo + 2;
3834   }
3835 
3836   return DefCycle;
3837 }
3838 
3839 bool ARMBaseInstrInfo::isLDMBaseRegInList(const MachineInstr &MI) const {
3840   Register BaseReg = MI.getOperand(0).getReg();
3841   for (unsigned i = 1, sz = MI.getNumOperands(); i < sz; ++i) {
3842     const auto &Op = MI.getOperand(i);
3843     if (Op.isReg() && Op.getReg() == BaseReg)
3844       return true;
3845   }
3846   return false;
3847 }
3848 unsigned
3849 ARMBaseInstrInfo::getLDMVariableDefsSize(const MachineInstr &MI) const {
3850   // ins GPR:$Rn, $p (2xOp), reglist:$regs, variable_ops
3851   // (outs GPR:$wb), (ins GPR:$Rn, $p (2xOp), reglist:$regs, variable_ops)
3852   return MI.getNumOperands() + 1 - MI.getDesc().getNumOperands();
3853 }
3854 
3855 int
3856 ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3857                                  const MCInstrDesc &DefMCID,
3858                                  unsigned DefClass,
3859                                  unsigned DefIdx, unsigned DefAlign) const {
3860   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3861   if (RegNo <= 0)
3862     // Def is the address writeback.
3863     return ItinData->getOperandCycle(DefClass, DefIdx);
3864 
3865   int DefCycle;
3866   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3867     // 4 registers would be issued: 1, 2, 1.
3868     // 5 registers would be issued: 1, 2, 2.
3869     DefCycle = RegNo / 2;
3870     if (DefCycle < 1)
3871       DefCycle = 1;
3872     // Result latency is issue cycle + 2: E2.
3873     DefCycle += 2;
3874   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3875     DefCycle = (RegNo / 2);
3876     // If there are odd number of registers or if it's not 64-bit aligned,
3877     // then it takes an extra AGU (Address Generation Unit) cycle.
3878     if ((RegNo % 2) || DefAlign < 8)
3879       ++DefCycle;
3880     // Result latency is AGU cycles + 2.
3881     DefCycle += 2;
3882   } else {
3883     // Assume the worst.
3884     DefCycle = RegNo + 2;
3885   }
3886 
3887   return DefCycle;
3888 }
3889 
3890 int
3891 ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3892                                   const MCInstrDesc &UseMCID,
3893                                   unsigned UseClass,
3894                                   unsigned UseIdx, unsigned UseAlign) const {
3895   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3896   if (RegNo <= 0)
3897     return ItinData->getOperandCycle(UseClass, UseIdx);
3898 
3899   int UseCycle;
3900   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3901     // (regno / 2) + (regno % 2) + 1
3902     UseCycle = RegNo / 2 + 1;
3903     if (RegNo % 2)
3904       ++UseCycle;
3905   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3906     UseCycle = RegNo;
3907     bool isSStore = false;
3908 
3909     switch (UseMCID.getOpcode()) {
3910     default: break;
3911     case ARM::VSTMSIA:
3912     case ARM::VSTMSIA_UPD:
3913     case ARM::VSTMSDB_UPD:
3914       isSStore = true;
3915       break;
3916     }
3917 
3918     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3919     // then it takes an extra cycle.
3920     if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3921       ++UseCycle;
3922   } else {
3923     // Assume the worst.
3924     UseCycle = RegNo + 2;
3925   }
3926 
3927   return UseCycle;
3928 }
3929 
3930 int
3931 ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3932                                  const MCInstrDesc &UseMCID,
3933                                  unsigned UseClass,
3934                                  unsigned UseIdx, unsigned UseAlign) const {
3935   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3936   if (RegNo <= 0)
3937     return ItinData->getOperandCycle(UseClass, UseIdx);
3938 
3939   int UseCycle;
3940   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3941     UseCycle = RegNo / 2;
3942     if (UseCycle < 2)
3943       UseCycle = 2;
3944     // Read in E3.
3945     UseCycle += 2;
3946   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3947     UseCycle = (RegNo / 2);
3948     // If there are odd number of registers or if it's not 64-bit aligned,
3949     // then it takes an extra AGU (Address Generation Unit) cycle.
3950     if ((RegNo % 2) || UseAlign < 8)
3951       ++UseCycle;
3952   } else {
3953     // Assume the worst.
3954     UseCycle = 1;
3955   }
3956   return UseCycle;
3957 }
3958 
3959 int
3960 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3961                                     const MCInstrDesc &DefMCID,
3962                                     unsigned DefIdx, unsigned DefAlign,
3963                                     const MCInstrDesc &UseMCID,
3964                                     unsigned UseIdx, unsigned UseAlign) const {
3965   unsigned DefClass = DefMCID.getSchedClass();
3966   unsigned UseClass = UseMCID.getSchedClass();
3967 
3968   if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3969     return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3970 
3971   // This may be a def / use of a variable_ops instruction, the operand
3972   // latency might be determinable dynamically. Let the target try to
3973   // figure it out.
3974   int DefCycle = -1;
3975   bool LdmBypass = false;
3976   switch (DefMCID.getOpcode()) {
3977   default:
3978     DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
3979     break;
3980 
3981   case ARM::VLDMDIA:
3982   case ARM::VLDMDIA_UPD:
3983   case ARM::VLDMDDB_UPD:
3984   case ARM::VLDMSIA:
3985   case ARM::VLDMSIA_UPD:
3986   case ARM::VLDMSDB_UPD:
3987     DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3988     break;
3989 
3990   case ARM::LDMIA_RET:
3991   case ARM::LDMIA:
3992   case ARM::LDMDA:
3993   case ARM::LDMDB:
3994   case ARM::LDMIB:
3995   case ARM::LDMIA_UPD:
3996   case ARM::LDMDA_UPD:
3997   case ARM::LDMDB_UPD:
3998   case ARM::LDMIB_UPD:
3999   case ARM::tLDMIA:
4000   case ARM::tLDMIA_UPD:
4001   case ARM::tPUSH:
4002   case ARM::t2LDMIA_RET:
4003   case ARM::t2LDMIA:
4004   case ARM::t2LDMDB:
4005   case ARM::t2LDMIA_UPD:
4006   case ARM::t2LDMDB_UPD:
4007     LdmBypass = true;
4008     DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4009     break;
4010   }
4011 
4012   if (DefCycle == -1)
4013     // We can't seem to determine the result latency of the def, assume it's 2.
4014     DefCycle = 2;
4015 
4016   int UseCycle = -1;
4017   switch (UseMCID.getOpcode()) {
4018   default:
4019     UseCycle = ItinData->getOperandCycle(UseClass, UseIdx);
4020     break;
4021 
4022   case ARM::VSTMDIA:
4023   case ARM::VSTMDIA_UPD:
4024   case ARM::VSTMDDB_UPD:
4025   case ARM::VSTMSIA:
4026   case ARM::VSTMSIA_UPD:
4027   case ARM::VSTMSDB_UPD:
4028     UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4029     break;
4030 
4031   case ARM::STMIA:
4032   case ARM::STMDA:
4033   case ARM::STMDB:
4034   case ARM::STMIB:
4035   case ARM::STMIA_UPD:
4036   case ARM::STMDA_UPD:
4037   case ARM::STMDB_UPD:
4038   case ARM::STMIB_UPD:
4039   case ARM::tSTMIA_UPD:
4040   case ARM::tPOP_RET:
4041   case ARM::tPOP:
4042   case ARM::t2STMIA:
4043   case ARM::t2STMDB:
4044   case ARM::t2STMIA_UPD:
4045   case ARM::t2STMDB_UPD:
4046     UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4047     break;
4048   }
4049 
4050   if (UseCycle == -1)
4051     // Assume it's read in the first stage.
4052     UseCycle = 1;
4053 
4054   UseCycle = DefCycle - UseCycle + 1;
4055   if (UseCycle > 0) {
4056     if (LdmBypass) {
4057       // It's a variable_ops instruction so we can't use DefIdx here. Just use
4058       // first def operand.
4059       if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1,
4060                                           UseClass, UseIdx))
4061         --UseCycle;
4062     } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
4063                                                UseClass, UseIdx)) {
4064       --UseCycle;
4065     }
4066   }
4067 
4068   return UseCycle;
4069 }
4070 
4071 static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
4072                                            const MachineInstr *MI, unsigned Reg,
4073                                            unsigned &DefIdx, unsigned &Dist) {
4074   Dist = 0;
4075 
4076   MachineBasicBlock::const_iterator I = MI; ++I;
4077   MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator());
4078   assert(II->isInsideBundle() && "Empty bundle?");
4079 
4080   int Idx = -1;
4081   while (II->isInsideBundle()) {
4082     Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI);
4083     if (Idx != -1)
4084       break;
4085     --II;
4086     ++Dist;
4087   }
4088 
4089   assert(Idx != -1 && "Cannot find bundled definition!");
4090   DefIdx = Idx;
4091   return &*II;
4092 }
4093 
4094 static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
4095                                            const MachineInstr &MI, unsigned Reg,
4096                                            unsigned &UseIdx, unsigned &Dist) {
4097   Dist = 0;
4098 
4099   MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
4100   assert(II->isInsideBundle() && "Empty bundle?");
4101   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4102 
4103   // FIXME: This doesn't properly handle multiple uses.
4104   int Idx = -1;
4105   while (II != E && II->isInsideBundle()) {
4106     Idx = II->findRegisterUseOperandIdx(Reg, false, TRI);
4107     if (Idx != -1)
4108       break;
4109     if (II->getOpcode() != ARM::t2IT)
4110       ++Dist;
4111     ++II;
4112   }
4113 
4114   if (Idx == -1) {
4115     Dist = 0;
4116     return nullptr;
4117   }
4118 
4119   UseIdx = Idx;
4120   return &*II;
4121 }
4122 
4123 /// Return the number of cycles to add to (or subtract from) the static
4124 /// itinerary based on the def opcode and alignment. The caller will ensure that
4125 /// adjusted latency is at least one cycle.
4126 static int adjustDefLatency(const ARMSubtarget &Subtarget,
4127                             const MachineInstr &DefMI,
4128                             const MCInstrDesc &DefMCID, unsigned DefAlign) {
4129   int Adjust = 0;
4130   if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4131     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4132     // variants are one cycle cheaper.
4133     switch (DefMCID.getOpcode()) {
4134     default: break;
4135     case ARM::LDRrs:
4136     case ARM::LDRBrs: {
4137       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4138       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4139       if (ShImm == 0 ||
4140           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4141         --Adjust;
4142       break;
4143     }
4144     case ARM::t2LDRs:
4145     case ARM::t2LDRBs:
4146     case ARM::t2LDRHs:
4147     case ARM::t2LDRSHs: {
4148       // Thumb2 mode: lsl only.
4149       unsigned ShAmt = DefMI.getOperand(3).getImm();
4150       if (ShAmt == 0 || ShAmt == 2)
4151         --Adjust;
4152       break;
4153     }
4154     }
4155   } else if (Subtarget.isSwift()) {
4156     // FIXME: Properly handle all of the latency adjustments for address
4157     // writeback.
4158     switch (DefMCID.getOpcode()) {
4159     default: break;
4160     case ARM::LDRrs:
4161     case ARM::LDRBrs: {
4162       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4163       bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
4164       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4165       if (!isSub &&
4166           (ShImm == 0 ||
4167            ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4168             ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
4169         Adjust -= 2;
4170       else if (!isSub &&
4171                ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4172         --Adjust;
4173       break;
4174     }
4175     case ARM::t2LDRs:
4176     case ARM::t2LDRBs:
4177     case ARM::t2LDRHs:
4178     case ARM::t2LDRSHs: {
4179       // Thumb2 mode: lsl only.
4180       unsigned ShAmt = DefMI.getOperand(3).getImm();
4181       if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4182         Adjust -= 2;
4183       break;
4184     }
4185     }
4186   }
4187 
4188   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4189     switch (DefMCID.getOpcode()) {
4190     default: break;
4191     case ARM::VLD1q8:
4192     case ARM::VLD1q16:
4193     case ARM::VLD1q32:
4194     case ARM::VLD1q64:
4195     case ARM::VLD1q8wb_fixed:
4196     case ARM::VLD1q16wb_fixed:
4197     case ARM::VLD1q32wb_fixed:
4198     case ARM::VLD1q64wb_fixed:
4199     case ARM::VLD1q8wb_register:
4200     case ARM::VLD1q16wb_register:
4201     case ARM::VLD1q32wb_register:
4202     case ARM::VLD1q64wb_register:
4203     case ARM::VLD2d8:
4204     case ARM::VLD2d16:
4205     case ARM::VLD2d32:
4206     case ARM::VLD2q8:
4207     case ARM::VLD2q16:
4208     case ARM::VLD2q32:
4209     case ARM::VLD2d8wb_fixed:
4210     case ARM::VLD2d16wb_fixed:
4211     case ARM::VLD2d32wb_fixed:
4212     case ARM::VLD2q8wb_fixed:
4213     case ARM::VLD2q16wb_fixed:
4214     case ARM::VLD2q32wb_fixed:
4215     case ARM::VLD2d8wb_register:
4216     case ARM::VLD2d16wb_register:
4217     case ARM::VLD2d32wb_register:
4218     case ARM::VLD2q8wb_register:
4219     case ARM::VLD2q16wb_register:
4220     case ARM::VLD2q32wb_register:
4221     case ARM::VLD3d8:
4222     case ARM::VLD3d16:
4223     case ARM::VLD3d32:
4224     case ARM::VLD1d64T:
4225     case ARM::VLD3d8_UPD:
4226     case ARM::VLD3d16_UPD:
4227     case ARM::VLD3d32_UPD:
4228     case ARM::VLD1d64Twb_fixed:
4229     case ARM::VLD1d64Twb_register:
4230     case ARM::VLD3q8_UPD:
4231     case ARM::VLD3q16_UPD:
4232     case ARM::VLD3q32_UPD:
4233     case ARM::VLD4d8:
4234     case ARM::VLD4d16:
4235     case ARM::VLD4d32:
4236     case ARM::VLD1d64Q:
4237     case ARM::VLD4d8_UPD:
4238     case ARM::VLD4d16_UPD:
4239     case ARM::VLD4d32_UPD:
4240     case ARM::VLD1d64Qwb_fixed:
4241     case ARM::VLD1d64Qwb_register:
4242     case ARM::VLD4q8_UPD:
4243     case ARM::VLD4q16_UPD:
4244     case ARM::VLD4q32_UPD:
4245     case ARM::VLD1DUPq8:
4246     case ARM::VLD1DUPq16:
4247     case ARM::VLD1DUPq32:
4248     case ARM::VLD1DUPq8wb_fixed:
4249     case ARM::VLD1DUPq16wb_fixed:
4250     case ARM::VLD1DUPq32wb_fixed:
4251     case ARM::VLD1DUPq8wb_register:
4252     case ARM::VLD1DUPq16wb_register:
4253     case ARM::VLD1DUPq32wb_register:
4254     case ARM::VLD2DUPd8:
4255     case ARM::VLD2DUPd16:
4256     case ARM::VLD2DUPd32:
4257     case ARM::VLD2DUPd8wb_fixed:
4258     case ARM::VLD2DUPd16wb_fixed:
4259     case ARM::VLD2DUPd32wb_fixed:
4260     case ARM::VLD2DUPd8wb_register:
4261     case ARM::VLD2DUPd16wb_register:
4262     case ARM::VLD2DUPd32wb_register:
4263     case ARM::VLD4DUPd8:
4264     case ARM::VLD4DUPd16:
4265     case ARM::VLD4DUPd32:
4266     case ARM::VLD4DUPd8_UPD:
4267     case ARM::VLD4DUPd16_UPD:
4268     case ARM::VLD4DUPd32_UPD:
4269     case ARM::VLD1LNd8:
4270     case ARM::VLD1LNd16:
4271     case ARM::VLD1LNd32:
4272     case ARM::VLD1LNd8_UPD:
4273     case ARM::VLD1LNd16_UPD:
4274     case ARM::VLD1LNd32_UPD:
4275     case ARM::VLD2LNd8:
4276     case ARM::VLD2LNd16:
4277     case ARM::VLD2LNd32:
4278     case ARM::VLD2LNq16:
4279     case ARM::VLD2LNq32:
4280     case ARM::VLD2LNd8_UPD:
4281     case ARM::VLD2LNd16_UPD:
4282     case ARM::VLD2LNd32_UPD:
4283     case ARM::VLD2LNq16_UPD:
4284     case ARM::VLD2LNq32_UPD:
4285     case ARM::VLD4LNd8:
4286     case ARM::VLD4LNd16:
4287     case ARM::VLD4LNd32:
4288     case ARM::VLD4LNq16:
4289     case ARM::VLD4LNq32:
4290     case ARM::VLD4LNd8_UPD:
4291     case ARM::VLD4LNd16_UPD:
4292     case ARM::VLD4LNd32_UPD:
4293     case ARM::VLD4LNq16_UPD:
4294     case ARM::VLD4LNq32_UPD:
4295       // If the address is not 64-bit aligned, the latencies of these
4296       // instructions increases by one.
4297       ++Adjust;
4298       break;
4299     }
4300   }
4301   return Adjust;
4302 }
4303 
4304 int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4305                                         const MachineInstr &DefMI,
4306                                         unsigned DefIdx,
4307                                         const MachineInstr &UseMI,
4308                                         unsigned UseIdx) const {
4309   // No operand latency. The caller may fall back to getInstrLatency.
4310   if (!ItinData || ItinData->isEmpty())
4311     return -1;
4312 
4313   const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
4314   Register Reg = DefMO.getReg();
4315 
4316   const MachineInstr *ResolvedDefMI = &DefMI;
4317   unsigned DefAdj = 0;
4318   if (DefMI.isBundle())
4319     ResolvedDefMI =
4320         getBundledDefMI(&getRegisterInfo(), &DefMI, Reg, DefIdx, DefAdj);
4321   if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4322       ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4323     return 1;
4324   }
4325 
4326   const MachineInstr *ResolvedUseMI = &UseMI;
4327   unsigned UseAdj = 0;
4328   if (UseMI.isBundle()) {
4329     ResolvedUseMI =
4330         getBundledUseMI(&getRegisterInfo(), UseMI, Reg, UseIdx, UseAdj);
4331     if (!ResolvedUseMI)
4332       return -1;
4333   }
4334 
4335   return getOperandLatencyImpl(
4336       ItinData, *ResolvedDefMI, DefIdx, ResolvedDefMI->getDesc(), DefAdj, DefMO,
4337       Reg, *ResolvedUseMI, UseIdx, ResolvedUseMI->getDesc(), UseAdj);
4338 }
4339 
4340 int ARMBaseInstrInfo::getOperandLatencyImpl(
4341     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4342     unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4343     const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4344     unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4345   if (Reg == ARM::CPSR) {
4346     if (DefMI.getOpcode() == ARM::FMSTAT) {
4347       // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4348       return Subtarget.isLikeA9() ? 1 : 20;
4349     }
4350 
4351     // CPSR set and branch can be paired in the same cycle.
4352     if (UseMI.isBranch())
4353       return 0;
4354 
4355     // Otherwise it takes the instruction latency (generally one).
4356     unsigned Latency = getInstrLatency(ItinData, DefMI);
4357 
4358     // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4359     // its uses. Instructions which are otherwise scheduled between them may
4360     // incur a code size penalty (not able to use the CPSR setting 16-bit
4361     // instructions).
4362     if (Latency > 0 && Subtarget.isThumb2()) {
4363       const MachineFunction *MF = DefMI.getParent()->getParent();
4364       // FIXME: Use Function::hasOptSize().
4365       if (MF->getFunction().hasFnAttribute(Attribute::OptimizeForSize))
4366         --Latency;
4367     }
4368     return Latency;
4369   }
4370 
4371   if (DefMO.isImplicit() || UseMI.getOperand(UseIdx).isImplicit())
4372     return -1;
4373 
4374   unsigned DefAlign = DefMI.hasOneMemOperand()
4375                           ? (*DefMI.memoperands_begin())->getAlign().value()
4376                           : 0;
4377   unsigned UseAlign = UseMI.hasOneMemOperand()
4378                           ? (*UseMI.memoperands_begin())->getAlign().value()
4379                           : 0;
4380 
4381   // Get the itinerary's latency if possible, and handle variable_ops.
4382   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, UseMCID,
4383                                   UseIdx, UseAlign);
4384   // Unable to find operand latency. The caller may resort to getInstrLatency.
4385   if (Latency < 0)
4386     return Latency;
4387 
4388   // Adjust for IT block position.
4389   int Adj = DefAdj + UseAdj;
4390 
4391   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4392   Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4393   if (Adj >= 0 || (int)Latency > -Adj) {
4394     return Latency + Adj;
4395   }
4396   // Return the itinerary latency, which may be zero but not less than zero.
4397   return Latency;
4398 }
4399 
4400 int
4401 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4402                                     SDNode *DefNode, unsigned DefIdx,
4403                                     SDNode *UseNode, unsigned UseIdx) const {
4404   if (!DefNode->isMachineOpcode())
4405     return 1;
4406 
4407   const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode());
4408 
4409   if (isZeroCost(DefMCID.Opcode))
4410     return 0;
4411 
4412   if (!ItinData || ItinData->isEmpty())
4413     return DefMCID.mayLoad() ? 3 : 1;
4414 
4415   if (!UseNode->isMachineOpcode()) {
4416     int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
4417     int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4418     int Threshold = 1 + Adj;
4419     return Latency <= Threshold ? 1 : Latency - Adj;
4420   }
4421 
4422   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
4423   auto *DefMN = cast<MachineSDNode>(DefNode);
4424   unsigned DefAlign = !DefMN->memoperands_empty()
4425                           ? (*DefMN->memoperands_begin())->getAlign().value()
4426                           : 0;
4427   auto *UseMN = cast<MachineSDNode>(UseNode);
4428   unsigned UseAlign = !UseMN->memoperands_empty()
4429                           ? (*UseMN->memoperands_begin())->getAlign().value()
4430                           : 0;
4431   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
4432                                   UseMCID, UseIdx, UseAlign);
4433 
4434   if (Latency > 1 &&
4435       (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4436        Subtarget.isCortexA7())) {
4437     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4438     // variants are one cycle cheaper.
4439     switch (DefMCID.getOpcode()) {
4440     default: break;
4441     case ARM::LDRrs:
4442     case ARM::LDRBrs: {
4443       unsigned ShOpVal =
4444         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4445       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4446       if (ShImm == 0 ||
4447           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4448         --Latency;
4449       break;
4450     }
4451     case ARM::t2LDRs:
4452     case ARM::t2LDRBs:
4453     case ARM::t2LDRHs:
4454     case ARM::t2LDRSHs: {
4455       // Thumb2 mode: lsl only.
4456       unsigned ShAmt =
4457         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4458       if (ShAmt == 0 || ShAmt == 2)
4459         --Latency;
4460       break;
4461     }
4462     }
4463   } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
4464     // FIXME: Properly handle all of the latency adjustments for address
4465     // writeback.
4466     switch (DefMCID.getOpcode()) {
4467     default: break;
4468     case ARM::LDRrs:
4469     case ARM::LDRBrs: {
4470       unsigned ShOpVal =
4471         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4472       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4473       if (ShImm == 0 ||
4474           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4475            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4476         Latency -= 2;
4477       else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4478         --Latency;
4479       break;
4480     }
4481     case ARM::t2LDRs:
4482     case ARM::t2LDRBs:
4483     case ARM::t2LDRHs:
4484     case ARM::t2LDRSHs:
4485       // Thumb2 mode: lsl 0-3 only.
4486       Latency -= 2;
4487       break;
4488     }
4489   }
4490 
4491   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4492     switch (DefMCID.getOpcode()) {
4493     default: break;
4494     case ARM::VLD1q8:
4495     case ARM::VLD1q16:
4496     case ARM::VLD1q32:
4497     case ARM::VLD1q64:
4498     case ARM::VLD1q8wb_register:
4499     case ARM::VLD1q16wb_register:
4500     case ARM::VLD1q32wb_register:
4501     case ARM::VLD1q64wb_register:
4502     case ARM::VLD1q8wb_fixed:
4503     case ARM::VLD1q16wb_fixed:
4504     case ARM::VLD1q32wb_fixed:
4505     case ARM::VLD1q64wb_fixed:
4506     case ARM::VLD2d8:
4507     case ARM::VLD2d16:
4508     case ARM::VLD2d32:
4509     case ARM::VLD2q8Pseudo:
4510     case ARM::VLD2q16Pseudo:
4511     case ARM::VLD2q32Pseudo:
4512     case ARM::VLD2d8wb_fixed:
4513     case ARM::VLD2d16wb_fixed:
4514     case ARM::VLD2d32wb_fixed:
4515     case ARM::VLD2q8PseudoWB_fixed:
4516     case ARM::VLD2q16PseudoWB_fixed:
4517     case ARM::VLD2q32PseudoWB_fixed:
4518     case ARM::VLD2d8wb_register:
4519     case ARM::VLD2d16wb_register:
4520     case ARM::VLD2d32wb_register:
4521     case ARM::VLD2q8PseudoWB_register:
4522     case ARM::VLD2q16PseudoWB_register:
4523     case ARM::VLD2q32PseudoWB_register:
4524     case ARM::VLD3d8Pseudo:
4525     case ARM::VLD3d16Pseudo:
4526     case ARM::VLD3d32Pseudo:
4527     case ARM::VLD1d8TPseudo:
4528     case ARM::VLD1d16TPseudo:
4529     case ARM::VLD1d32TPseudo:
4530     case ARM::VLD1d64TPseudo:
4531     case ARM::VLD1d64TPseudoWB_fixed:
4532     case ARM::VLD1d64TPseudoWB_register:
4533     case ARM::VLD3d8Pseudo_UPD:
4534     case ARM::VLD3d16Pseudo_UPD:
4535     case ARM::VLD3d32Pseudo_UPD:
4536     case ARM::VLD3q8Pseudo_UPD:
4537     case ARM::VLD3q16Pseudo_UPD:
4538     case ARM::VLD3q32Pseudo_UPD:
4539     case ARM::VLD3q8oddPseudo:
4540     case ARM::VLD3q16oddPseudo:
4541     case ARM::VLD3q32oddPseudo:
4542     case ARM::VLD3q8oddPseudo_UPD:
4543     case ARM::VLD3q16oddPseudo_UPD:
4544     case ARM::VLD3q32oddPseudo_UPD:
4545     case ARM::VLD4d8Pseudo:
4546     case ARM::VLD4d16Pseudo:
4547     case ARM::VLD4d32Pseudo:
4548     case ARM::VLD1d8QPseudo:
4549     case ARM::VLD1d16QPseudo:
4550     case ARM::VLD1d32QPseudo:
4551     case ARM::VLD1d64QPseudo:
4552     case ARM::VLD1d64QPseudoWB_fixed:
4553     case ARM::VLD1d64QPseudoWB_register:
4554     case ARM::VLD1q8HighQPseudo:
4555     case ARM::VLD1q8LowQPseudo_UPD:
4556     case ARM::VLD1q8HighTPseudo:
4557     case ARM::VLD1q8LowTPseudo_UPD:
4558     case ARM::VLD1q16HighQPseudo:
4559     case ARM::VLD1q16LowQPseudo_UPD:
4560     case ARM::VLD1q16HighTPseudo:
4561     case ARM::VLD1q16LowTPseudo_UPD:
4562     case ARM::VLD1q32HighQPseudo:
4563     case ARM::VLD1q32LowQPseudo_UPD:
4564     case ARM::VLD1q32HighTPseudo:
4565     case ARM::VLD1q32LowTPseudo_UPD:
4566     case ARM::VLD1q64HighQPseudo:
4567     case ARM::VLD1q64LowQPseudo_UPD:
4568     case ARM::VLD1q64HighTPseudo:
4569     case ARM::VLD1q64LowTPseudo_UPD:
4570     case ARM::VLD4d8Pseudo_UPD:
4571     case ARM::VLD4d16Pseudo_UPD:
4572     case ARM::VLD4d32Pseudo_UPD:
4573     case ARM::VLD4q8Pseudo_UPD:
4574     case ARM::VLD4q16Pseudo_UPD:
4575     case ARM::VLD4q32Pseudo_UPD:
4576     case ARM::VLD4q8oddPseudo:
4577     case ARM::VLD4q16oddPseudo:
4578     case ARM::VLD4q32oddPseudo:
4579     case ARM::VLD4q8oddPseudo_UPD:
4580     case ARM::VLD4q16oddPseudo_UPD:
4581     case ARM::VLD4q32oddPseudo_UPD:
4582     case ARM::VLD1DUPq8:
4583     case ARM::VLD1DUPq16:
4584     case ARM::VLD1DUPq32:
4585     case ARM::VLD1DUPq8wb_fixed:
4586     case ARM::VLD1DUPq16wb_fixed:
4587     case ARM::VLD1DUPq32wb_fixed:
4588     case ARM::VLD1DUPq8wb_register:
4589     case ARM::VLD1DUPq16wb_register:
4590     case ARM::VLD1DUPq32wb_register:
4591     case ARM::VLD2DUPd8:
4592     case ARM::VLD2DUPd16:
4593     case ARM::VLD2DUPd32:
4594     case ARM::VLD2DUPd8wb_fixed:
4595     case ARM::VLD2DUPd16wb_fixed:
4596     case ARM::VLD2DUPd32wb_fixed:
4597     case ARM::VLD2DUPd8wb_register:
4598     case ARM::VLD2DUPd16wb_register:
4599     case ARM::VLD2DUPd32wb_register:
4600     case ARM::VLD2DUPq8EvenPseudo:
4601     case ARM::VLD2DUPq8OddPseudo:
4602     case ARM::VLD2DUPq16EvenPseudo:
4603     case ARM::VLD2DUPq16OddPseudo:
4604     case ARM::VLD2DUPq32EvenPseudo:
4605     case ARM::VLD2DUPq32OddPseudo:
4606     case ARM::VLD3DUPq8EvenPseudo:
4607     case ARM::VLD3DUPq8OddPseudo:
4608     case ARM::VLD3DUPq16EvenPseudo:
4609     case ARM::VLD3DUPq16OddPseudo:
4610     case ARM::VLD3DUPq32EvenPseudo:
4611     case ARM::VLD3DUPq32OddPseudo:
4612     case ARM::VLD4DUPd8Pseudo:
4613     case ARM::VLD4DUPd16Pseudo:
4614     case ARM::VLD4DUPd32Pseudo:
4615     case ARM::VLD4DUPd8Pseudo_UPD:
4616     case ARM::VLD4DUPd16Pseudo_UPD:
4617     case ARM::VLD4DUPd32Pseudo_UPD:
4618     case ARM::VLD4DUPq8EvenPseudo:
4619     case ARM::VLD4DUPq8OddPseudo:
4620     case ARM::VLD4DUPq16EvenPseudo:
4621     case ARM::VLD4DUPq16OddPseudo:
4622     case ARM::VLD4DUPq32EvenPseudo:
4623     case ARM::VLD4DUPq32OddPseudo:
4624     case ARM::VLD1LNq8Pseudo:
4625     case ARM::VLD1LNq16Pseudo:
4626     case ARM::VLD1LNq32Pseudo:
4627     case ARM::VLD1LNq8Pseudo_UPD:
4628     case ARM::VLD1LNq16Pseudo_UPD:
4629     case ARM::VLD1LNq32Pseudo_UPD:
4630     case ARM::VLD2LNd8Pseudo:
4631     case ARM::VLD2LNd16Pseudo:
4632     case ARM::VLD2LNd32Pseudo:
4633     case ARM::VLD2LNq16Pseudo:
4634     case ARM::VLD2LNq32Pseudo:
4635     case ARM::VLD2LNd8Pseudo_UPD:
4636     case ARM::VLD2LNd16Pseudo_UPD:
4637     case ARM::VLD2LNd32Pseudo_UPD:
4638     case ARM::VLD2LNq16Pseudo_UPD:
4639     case ARM::VLD2LNq32Pseudo_UPD:
4640     case ARM::VLD4LNd8Pseudo:
4641     case ARM::VLD4LNd16Pseudo:
4642     case ARM::VLD4LNd32Pseudo:
4643     case ARM::VLD4LNq16Pseudo:
4644     case ARM::VLD4LNq32Pseudo:
4645     case ARM::VLD4LNd8Pseudo_UPD:
4646     case ARM::VLD4LNd16Pseudo_UPD:
4647     case ARM::VLD4LNd32Pseudo_UPD:
4648     case ARM::VLD4LNq16Pseudo_UPD:
4649     case ARM::VLD4LNq32Pseudo_UPD:
4650       // If the address is not 64-bit aligned, the latencies of these
4651       // instructions increases by one.
4652       ++Latency;
4653       break;
4654     }
4655 
4656   return Latency;
4657 }
4658 
4659 unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4660   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4661       MI.isImplicitDef())
4662     return 0;
4663 
4664   if (MI.isBundle())
4665     return 0;
4666 
4667   const MCInstrDesc &MCID = MI.getDesc();
4668 
4669   if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4670                         !Subtarget.cheapPredicableCPSRDef())) {
4671     // When predicated, CPSR is an additional source operand for CPSR updating
4672     // instructions, this apparently increases their latencies.
4673     return 1;
4674   }
4675   return 0;
4676 }
4677 
4678 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4679                                            const MachineInstr &MI,
4680                                            unsigned *PredCost) const {
4681   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4682       MI.isImplicitDef())
4683     return 1;
4684 
4685   // An instruction scheduler typically runs on unbundled instructions, however
4686   // other passes may query the latency of a bundled instruction.
4687   if (MI.isBundle()) {
4688     unsigned Latency = 0;
4689     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4690     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4691     while (++I != E && I->isInsideBundle()) {
4692       if (I->getOpcode() != ARM::t2IT)
4693         Latency += getInstrLatency(ItinData, *I, PredCost);
4694     }
4695     return Latency;
4696   }
4697 
4698   const MCInstrDesc &MCID = MI.getDesc();
4699   if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4700                                      !Subtarget.cheapPredicableCPSRDef()))) {
4701     // When predicated, CPSR is an additional source operand for CPSR updating
4702     // instructions, this apparently increases their latencies.
4703     *PredCost = 1;
4704   }
4705   // Be sure to call getStageLatency for an empty itinerary in case it has a
4706   // valid MinLatency property.
4707   if (!ItinData)
4708     return MI.mayLoad() ? 3 : 1;
4709 
4710   unsigned Class = MCID.getSchedClass();
4711 
4712   // For instructions with variable uops, use uops as latency.
4713   if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0)
4714     return getNumMicroOps(ItinData, MI);
4715 
4716   // For the common case, fall back on the itinerary's latency.
4717   unsigned Latency = ItinData->getStageLatency(Class);
4718 
4719   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4720   unsigned DefAlign =
4721       MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4722   int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign);
4723   if (Adj >= 0 || (int)Latency > -Adj) {
4724     return Latency + Adj;
4725   }
4726   return Latency;
4727 }
4728 
4729 int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4730                                       SDNode *Node) const {
4731   if (!Node->isMachineOpcode())
4732     return 1;
4733 
4734   if (!ItinData || ItinData->isEmpty())
4735     return 1;
4736 
4737   unsigned Opcode = Node->getMachineOpcode();
4738   switch (Opcode) {
4739   default:
4740     return ItinData->getStageLatency(get(Opcode).getSchedClass());
4741   case ARM::VLDMQIA:
4742   case ARM::VSTMQIA:
4743     return 2;
4744   }
4745 }
4746 
4747 bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4748                                              const MachineRegisterInfo *MRI,
4749                                              const MachineInstr &DefMI,
4750                                              unsigned DefIdx,
4751                                              const MachineInstr &UseMI,
4752                                              unsigned UseIdx) const {
4753   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4754   unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4755   if (Subtarget.nonpipelinedVFP() &&
4756       (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4757     return true;
4758 
4759   // Hoist VFP / NEON instructions with 4 or higher latency.
4760   unsigned Latency =
4761       SchedModel.computeOperandLatency(&DefMI, DefIdx, &UseMI, UseIdx);
4762   if (Latency <= 3)
4763     return false;
4764   return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4765          UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4766 }
4767 
4768 bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4769                                         const MachineInstr &DefMI,
4770                                         unsigned DefIdx) const {
4771   const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4772   if (!ItinData || ItinData->isEmpty())
4773     return false;
4774 
4775   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4776   if (DDomain == ARMII::DomainGeneral) {
4777     unsigned DefClass = DefMI.getDesc().getSchedClass();
4778     int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4779     return (DefCycle != -1 && DefCycle <= 2);
4780   }
4781   return false;
4782 }
4783 
4784 bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4785                                          StringRef &ErrInfo) const {
4786   if (convertAddSubFlagsOpcode(MI.getOpcode())) {
4787     ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4788     return false;
4789   }
4790   if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4791     // Make sure we don't generate a lo-lo mov that isn't supported.
4792     if (!ARM::hGPRRegClass.contains(MI.getOperand(0).getReg()) &&
4793         !ARM::hGPRRegClass.contains(MI.getOperand(1).getReg())) {
4794       ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4795       return false;
4796     }
4797   }
4798   if (MI.getOpcode() == ARM::tPUSH ||
4799       MI.getOpcode() == ARM::tPOP ||
4800       MI.getOpcode() == ARM::tPOP_RET) {
4801     for (int i = 2, e = MI.getNumOperands(); i < e; ++i) {
4802       if (MI.getOperand(i).isImplicit() ||
4803           !MI.getOperand(i).isReg())
4804         continue;
4805       Register Reg = MI.getOperand(i).getReg();
4806       if (Reg < ARM::R0 || Reg > ARM::R7) {
4807         if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4808             !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4809           ErrInfo = "Unsupported register in Thumb1 push/pop";
4810           return false;
4811         }
4812       }
4813     }
4814   }
4815   return true;
4816 }
4817 
4818 // LoadStackGuard has so far only been implemented for MachO. Different code
4819 // sequence is needed for other targets.
4820 void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4821                                                 unsigned LoadImmOpc,
4822                                                 unsigned LoadOpc) const {
4823   assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4824          "ROPI/RWPI not currently supported with stack guard");
4825 
4826   MachineBasicBlock &MBB = *MI->getParent();
4827   DebugLoc DL = MI->getDebugLoc();
4828   Register Reg = MI->getOperand(0).getReg();
4829   const GlobalValue *GV =
4830       cast<GlobalValue>((*MI->memoperands_begin())->getValue());
4831   MachineInstrBuilder MIB;
4832 
4833   BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4834       .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY);
4835 
4836   if (Subtarget.isGVIndirectSymbol(GV)) {
4837     MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4838     MIB.addReg(Reg, RegState::Kill).addImm(0);
4839     auto Flags = MachineMemOperand::MOLoad |
4840                  MachineMemOperand::MODereferenceable |
4841                  MachineMemOperand::MOInvariant;
4842     MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4843         MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
4844     MIB.addMemOperand(MMO).add(predOps(ARMCC::AL));
4845   }
4846 
4847   MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4848   MIB.addReg(Reg, RegState::Kill)
4849       .addImm(0)
4850       .cloneMemRefs(*MI)
4851       .add(predOps(ARMCC::AL));
4852 }
4853 
4854 bool
4855 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4856                                      unsigned &AddSubOpc,
4857                                      bool &NegAcc, bool &HasLane) const {
4858   DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode);
4859   if (I == MLxEntryMap.end())
4860     return false;
4861 
4862   const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4863   MulOpc = Entry.MulOpc;
4864   AddSubOpc = Entry.AddSubOpc;
4865   NegAcc = Entry.NegAcc;
4866   HasLane = Entry.HasLane;
4867   return true;
4868 }
4869 
4870 //===----------------------------------------------------------------------===//
4871 // Execution domains.
4872 //===----------------------------------------------------------------------===//
4873 //
4874 // Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4875 // and some can go down both.  The vmov instructions go down the VFP pipeline,
4876 // but they can be changed to vorr equivalents that are executed by the NEON
4877 // pipeline.
4878 //
4879 // We use the following execution domain numbering:
4880 //
4881 enum ARMExeDomain {
4882   ExeGeneric = 0,
4883   ExeVFP = 1,
4884   ExeNEON = 2
4885 };
4886 
4887 //
4888 // Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4889 //
4890 std::pair<uint16_t, uint16_t>
4891 ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
4892   // If we don't have access to NEON instructions then we won't be able
4893   // to swizzle anything to the NEON domain. Check to make sure.
4894   if (Subtarget.hasNEON()) {
4895     // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4896     // if they are not predicated.
4897     if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
4898       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4899 
4900     // CortexA9 is particularly picky about mixing the two and wants these
4901     // converted.
4902     if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
4903         (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
4904          MI.getOpcode() == ARM::VMOVS))
4905       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4906   }
4907   // No other instructions can be swizzled, so just determine their domain.
4908   unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
4909 
4910   if (Domain & ARMII::DomainNEON)
4911     return std::make_pair(ExeNEON, 0);
4912 
4913   // Certain instructions can go either way on Cortex-A8.
4914   // Treat them as NEON instructions.
4915   if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
4916     return std::make_pair(ExeNEON, 0);
4917 
4918   if (Domain & ARMII::DomainVFP)
4919     return std::make_pair(ExeVFP, 0);
4920 
4921   return std::make_pair(ExeGeneric, 0);
4922 }
4923 
4924 static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
4925                                             unsigned SReg, unsigned &Lane) {
4926   unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
4927   Lane = 0;
4928 
4929   if (DReg != ARM::NoRegister)
4930    return DReg;
4931 
4932   Lane = 1;
4933   DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
4934 
4935   assert(DReg && "S-register with no D super-register?");
4936   return DReg;
4937 }
4938 
4939 /// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
4940 /// set ImplicitSReg to a register number that must be marked as implicit-use or
4941 /// zero if no register needs to be defined as implicit-use.
4942 ///
4943 /// If the function cannot determine if an SPR should be marked implicit use or
4944 /// not, it returns false.
4945 ///
4946 /// This function handles cases where an instruction is being modified from taking
4947 /// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
4948 /// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
4949 /// lane of the DPR).
4950 ///
4951 /// If the other SPR is defined, an implicit-use of it should be added. Else,
4952 /// (including the case where the DPR itself is defined), it should not.
4953 ///
4954 static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
4955                                        MachineInstr &MI, unsigned DReg,
4956                                        unsigned Lane, unsigned &ImplicitSReg) {
4957   // If the DPR is defined or used already, the other SPR lane will be chained
4958   // correctly, so there is nothing to be done.
4959   if (MI.definesRegister(DReg, TRI) || MI.readsRegister(DReg, TRI)) {
4960     ImplicitSReg = 0;
4961     return true;
4962   }
4963 
4964   // Otherwise we need to go searching to see if the SPR is set explicitly.
4965   ImplicitSReg = TRI->getSubReg(DReg,
4966                                 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
4967   MachineBasicBlock::LivenessQueryResult LQR =
4968       MI.getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
4969 
4970   if (LQR == MachineBasicBlock::LQR_Live)
4971     return true;
4972   else if (LQR == MachineBasicBlock::LQR_Unknown)
4973     return false;
4974 
4975   // If the register is known not to be live, there is no need to add an
4976   // implicit-use.
4977   ImplicitSReg = 0;
4978   return true;
4979 }
4980 
4981 void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
4982                                           unsigned Domain) const {
4983   unsigned DstReg, SrcReg, DReg;
4984   unsigned Lane;
4985   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
4986   const TargetRegisterInfo *TRI = &getRegisterInfo();
4987   switch (MI.getOpcode()) {
4988   default:
4989     llvm_unreachable("cannot handle opcode!");
4990     break;
4991   case ARM::VMOVD:
4992     if (Domain != ExeNEON)
4993       break;
4994 
4995     // Zap the predicate operands.
4996     assert(!isPredicated(MI) && "Cannot predicate a VORRd");
4997 
4998     // Make sure we've got NEON instructions.
4999     assert(Subtarget.hasNEON() && "VORRd requires NEON");
5000 
5001     // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
5002     DstReg = MI.getOperand(0).getReg();
5003     SrcReg = MI.getOperand(1).getReg();
5004 
5005     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5006       MI.RemoveOperand(i - 1);
5007 
5008     // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
5009     MI.setDesc(get(ARM::VORRd));
5010     MIB.addReg(DstReg, RegState::Define)
5011         .addReg(SrcReg)
5012         .addReg(SrcReg)
5013         .add(predOps(ARMCC::AL));
5014     break;
5015   case ARM::VMOVRS:
5016     if (Domain != ExeNEON)
5017       break;
5018     assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
5019 
5020     // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
5021     DstReg = MI.getOperand(0).getReg();
5022     SrcReg = MI.getOperand(1).getReg();
5023 
5024     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5025       MI.RemoveOperand(i - 1);
5026 
5027     DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
5028 
5029     // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5030     // Note that DSrc has been widened and the other lane may be undef, which
5031     // contaminates the entire register.
5032     MI.setDesc(get(ARM::VGETLNi32));
5033     MIB.addReg(DstReg, RegState::Define)
5034         .addReg(DReg, RegState::Undef)
5035         .addImm(Lane)
5036         .add(predOps(ARMCC::AL));
5037 
5038     // The old source should be an implicit use, otherwise we might think it
5039     // was dead before here.
5040     MIB.addReg(SrcReg, RegState::Implicit);
5041     break;
5042   case ARM::VMOVSR: {
5043     if (Domain != ExeNEON)
5044       break;
5045     assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5046 
5047     // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5048     DstReg = MI.getOperand(0).getReg();
5049     SrcReg = MI.getOperand(1).getReg();
5050 
5051     DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
5052 
5053     unsigned ImplicitSReg;
5054     if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5055       break;
5056 
5057     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5058       MI.RemoveOperand(i - 1);
5059 
5060     // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5061     // Again DDst may be undefined at the beginning of this instruction.
5062     MI.setDesc(get(ARM::VSETLNi32));
5063     MIB.addReg(DReg, RegState::Define)
5064         .addReg(DReg, getUndefRegState(!MI.readsRegister(DReg, TRI)))
5065         .addReg(SrcReg)
5066         .addImm(Lane)
5067         .add(predOps(ARMCC::AL));
5068 
5069     // The narrower destination must be marked as set to keep previous chains
5070     // in place.
5071     MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5072     if (ImplicitSReg != 0)
5073       MIB.addReg(ImplicitSReg, RegState::Implicit);
5074     break;
5075     }
5076     case ARM::VMOVS: {
5077       if (Domain != ExeNEON)
5078         break;
5079 
5080       // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5081       DstReg = MI.getOperand(0).getReg();
5082       SrcReg = MI.getOperand(1).getReg();
5083 
5084       unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
5085       DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
5086       DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
5087 
5088       unsigned ImplicitSReg;
5089       if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
5090         break;
5091 
5092       for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5093         MI.RemoveOperand(i - 1);
5094 
5095       if (DSrc == DDst) {
5096         // Destination can be:
5097         //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5098         MI.setDesc(get(ARM::VDUPLN32d));
5099         MIB.addReg(DDst, RegState::Define)
5100             .addReg(DDst, getUndefRegState(!MI.readsRegister(DDst, TRI)))
5101             .addImm(SrcLane)
5102             .add(predOps(ARMCC::AL));
5103 
5104         // Neither the source or the destination are naturally represented any
5105         // more, so add them in manually.
5106         MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
5107         MIB.addReg(SrcReg, RegState::Implicit);
5108         if (ImplicitSReg != 0)
5109           MIB.addReg(ImplicitSReg, RegState::Implicit);
5110         break;
5111       }
5112 
5113       // In general there's no single instruction that can perform an S <-> S
5114       // move in NEON space, but a pair of VEXT instructions *can* do the
5115       // job. It turns out that the VEXTs needed will only use DSrc once, with
5116       // the position based purely on the combination of lane-0 and lane-1
5117       // involved. For example
5118       //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
5119       //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
5120       //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
5121       //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
5122       //
5123       // Pattern of the MachineInstrs is:
5124       //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5125       MachineInstrBuilder NewMIB;
5126       NewMIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::VEXTd32),
5127                        DDst);
5128 
5129       // On the first instruction, both DSrc and DDst may be undef if present.
5130       // Specifically when the original instruction didn't have them as an
5131       // <imp-use>.
5132       unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5133       bool CurUndef = !MI.readsRegister(CurReg, TRI);
5134       NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
5135 
5136       CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5137       CurUndef = !MI.readsRegister(CurReg, TRI);
5138       NewMIB.addReg(CurReg, getUndefRegState(CurUndef))
5139             .addImm(1)
5140             .add(predOps(ARMCC::AL));
5141 
5142       if (SrcLane == DstLane)
5143         NewMIB.addReg(SrcReg, RegState::Implicit);
5144 
5145       MI.setDesc(get(ARM::VEXTd32));
5146       MIB.addReg(DDst, RegState::Define);
5147 
5148       // On the second instruction, DDst has definitely been defined above, so
5149       // it is not undef. DSrc, if present, can be undef as above.
5150       CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5151       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5152       MIB.addReg(CurReg, getUndefRegState(CurUndef));
5153 
5154       CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5155       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5156       MIB.addReg(CurReg, getUndefRegState(CurUndef))
5157          .addImm(1)
5158          .add(predOps(ARMCC::AL));
5159 
5160       if (SrcLane != DstLane)
5161         MIB.addReg(SrcReg, RegState::Implicit);
5162 
5163       // As before, the original destination is no longer represented, add it
5164       // implicitly.
5165       MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5166       if (ImplicitSReg != 0)
5167         MIB.addReg(ImplicitSReg, RegState::Implicit);
5168       break;
5169     }
5170   }
5171 }
5172 
5173 //===----------------------------------------------------------------------===//
5174 // Partial register updates
5175 //===----------------------------------------------------------------------===//
5176 //
5177 // Swift renames NEON registers with 64-bit granularity.  That means any
5178 // instruction writing an S-reg implicitly reads the containing D-reg.  The
5179 // problem is mostly avoided by translating f32 operations to v2f32 operations
5180 // on D-registers, but f32 loads are still a problem.
5181 //
5182 // These instructions can load an f32 into a NEON register:
5183 //
5184 // VLDRS - Only writes S, partial D update.
5185 // VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5186 // VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5187 //
5188 // FCONSTD can be used as a dependency-breaking instruction.
5189 unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5190     const MachineInstr &MI, unsigned OpNum,
5191     const TargetRegisterInfo *TRI) const {
5192   auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5193   if (!PartialUpdateClearance)
5194     return 0;
5195 
5196   assert(TRI && "Need TRI instance");
5197 
5198   const MachineOperand &MO = MI.getOperand(OpNum);
5199   if (MO.readsReg())
5200     return 0;
5201   Register Reg = MO.getReg();
5202   int UseOp = -1;
5203 
5204   switch (MI.getOpcode()) {
5205   // Normal instructions writing only an S-register.
5206   case ARM::VLDRS:
5207   case ARM::FCONSTS:
5208   case ARM::VMOVSR:
5209   case ARM::VMOVv8i8:
5210   case ARM::VMOVv4i16:
5211   case ARM::VMOVv2i32:
5212   case ARM::VMOVv2f32:
5213   case ARM::VMOVv1i64:
5214     UseOp = MI.findRegisterUseOperandIdx(Reg, false, TRI);
5215     break;
5216 
5217     // Explicitly reads the dependency.
5218   case ARM::VLD1LNd32:
5219     UseOp = 3;
5220     break;
5221   default:
5222     return 0;
5223   }
5224 
5225   // If this instruction actually reads a value from Reg, there is no unwanted
5226   // dependency.
5227   if (UseOp != -1 && MI.getOperand(UseOp).readsReg())
5228     return 0;
5229 
5230   // We must be able to clobber the whole D-reg.
5231   if (Register::isVirtualRegister(Reg)) {
5232     // Virtual register must be a def undef foo:ssub_0 operand.
5233     if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5234       return 0;
5235   } else if (ARM::SPRRegClass.contains(Reg)) {
5236     // Physical register: MI must define the full D-reg.
5237     unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
5238                                              &ARM::DPRRegClass);
5239     if (!DReg || !MI.definesRegister(DReg, TRI))
5240       return 0;
5241   }
5242 
5243   // MI has an unwanted D-register dependency.
5244   // Avoid defs in the previous N instructrions.
5245   return PartialUpdateClearance;
5246 }
5247 
5248 // Break a partial register dependency after getPartialRegUpdateClearance
5249 // returned non-zero.
5250 void ARMBaseInstrInfo::breakPartialRegDependency(
5251     MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5252   assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5253   assert(TRI && "Need TRI instance");
5254 
5255   const MachineOperand &MO = MI.getOperand(OpNum);
5256   Register Reg = MO.getReg();
5257   assert(Register::isPhysicalRegister(Reg) &&
5258          "Can't break virtual register dependencies.");
5259   unsigned DReg = Reg;
5260 
5261   // If MI defines an S-reg, find the corresponding D super-register.
5262   if (ARM::SPRRegClass.contains(Reg)) {
5263     DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5264     assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5265   }
5266 
5267   assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5268   assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5269 
5270   // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5271   // the full D-register by loading the same value to both lanes.  The
5272   // instruction is micro-coded with 2 uops, so don't do this until we can
5273   // properly schedule micro-coded instructions.  The dispatcher stalls cause
5274   // too big regressions.
5275 
5276   // Insert the dependency-breaking FCONSTD before MI.
5277   // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5278   BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg)
5279       .addImm(96)
5280       .add(predOps(ARMCC::AL));
5281   MI.addRegisterKilled(DReg, TRI, true);
5282 }
5283 
5284 bool ARMBaseInstrInfo::hasNOP() const {
5285   return Subtarget.getFeatureBits()[ARM::HasV6KOps];
5286 }
5287 
5288 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5289   if (MI->getNumOperands() < 4)
5290     return true;
5291   unsigned ShOpVal = MI->getOperand(3).getImm();
5292   unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal);
5293   // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5294   if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) ||
5295       ((ShImm == 1 || ShImm == 2) &&
5296        ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl))
5297     return true;
5298 
5299   return false;
5300 }
5301 
5302 bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5303     const MachineInstr &MI, unsigned DefIdx,
5304     SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5305   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5306   assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5307 
5308   switch (MI.getOpcode()) {
5309   case ARM::VMOVDRR:
5310     // dX = VMOVDRR rY, rZ
5311     // is the same as:
5312     // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5313     // Populate the InputRegs accordingly.
5314     // rY
5315     const MachineOperand *MOReg = &MI.getOperand(1);
5316     if (!MOReg->isUndef())
5317       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5318                                               MOReg->getSubReg(), ARM::ssub_0));
5319     // rZ
5320     MOReg = &MI.getOperand(2);
5321     if (!MOReg->isUndef())
5322       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5323                                               MOReg->getSubReg(), ARM::ssub_1));
5324     return true;
5325   }
5326   llvm_unreachable("Target dependent opcode missing");
5327 }
5328 
5329 bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5330     const MachineInstr &MI, unsigned DefIdx,
5331     RegSubRegPairAndIdx &InputReg) const {
5332   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5333   assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5334 
5335   switch (MI.getOpcode()) {
5336   case ARM::VMOVRRD:
5337     // rX, rY = VMOVRRD dZ
5338     // is the same as:
5339     // rX = EXTRACT_SUBREG dZ, ssub_0
5340     // rY = EXTRACT_SUBREG dZ, ssub_1
5341     const MachineOperand &MOReg = MI.getOperand(2);
5342     if (MOReg.isUndef())
5343       return false;
5344     InputReg.Reg = MOReg.getReg();
5345     InputReg.SubReg = MOReg.getSubReg();
5346     InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5347     return true;
5348   }
5349   llvm_unreachable("Target dependent opcode missing");
5350 }
5351 
5352 bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5353     const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5354     RegSubRegPairAndIdx &InsertedReg) const {
5355   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5356   assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5357 
5358   switch (MI.getOpcode()) {
5359   case ARM::VSETLNi32:
5360     // dX = VSETLNi32 dY, rZ, imm
5361     const MachineOperand &MOBaseReg = MI.getOperand(1);
5362     const MachineOperand &MOInsertedReg = MI.getOperand(2);
5363     if (MOInsertedReg.isUndef())
5364       return false;
5365     const MachineOperand &MOIndex = MI.getOperand(3);
5366     BaseReg.Reg = MOBaseReg.getReg();
5367     BaseReg.SubReg = MOBaseReg.getSubReg();
5368 
5369     InsertedReg.Reg = MOInsertedReg.getReg();
5370     InsertedReg.SubReg = MOInsertedReg.getSubReg();
5371     InsertedReg.SubIdx = MOIndex.getImm() == 0 ? ARM::ssub_0 : ARM::ssub_1;
5372     return true;
5373   }
5374   llvm_unreachable("Target dependent opcode missing");
5375 }
5376 
5377 std::pair<unsigned, unsigned>
5378 ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5379   const unsigned Mask = ARMII::MO_OPTION_MASK;
5380   return std::make_pair(TF & Mask, TF & ~Mask);
5381 }
5382 
5383 ArrayRef<std::pair<unsigned, const char *>>
5384 ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5385   using namespace ARMII;
5386 
5387   static const std::pair<unsigned, const char *> TargetFlags[] = {
5388       {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"}};
5389   return makeArrayRef(TargetFlags);
5390 }
5391 
5392 ArrayRef<std::pair<unsigned, const char *>>
5393 ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5394   using namespace ARMII;
5395 
5396   static const std::pair<unsigned, const char *> TargetFlags[] = {
5397       {MO_COFFSTUB, "arm-coffstub"},
5398       {MO_GOT, "arm-got"},
5399       {MO_SBREL, "arm-sbrel"},
5400       {MO_DLLIMPORT, "arm-dllimport"},
5401       {MO_SECREL, "arm-secrel"},
5402       {MO_NONLAZY, "arm-nonlazy"}};
5403   return makeArrayRef(TargetFlags);
5404 }
5405 
5406 Optional<RegImmPair> ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI,
5407                                                       Register Reg) const {
5408   int Sign = 1;
5409   unsigned Opcode = MI.getOpcode();
5410   int64_t Offset = 0;
5411 
5412   // TODO: Handle cases where Reg is a super- or sub-register of the
5413   // destination register.
5414   const MachineOperand &Op0 = MI.getOperand(0);
5415   if (!Op0.isReg() || Reg != Op0.getReg())
5416     return None;
5417 
5418   // We describe SUBri or ADDri instructions.
5419   if (Opcode == ARM::SUBri)
5420     Sign = -1;
5421   else if (Opcode != ARM::ADDri)
5422     return None;
5423 
5424   // TODO: Third operand can be global address (usually some string). Since
5425   //       strings can be relocated we cannot calculate their offsets for
5426   //       now.
5427   if (!MI.getOperand(1).isReg() || !MI.getOperand(2).isImm())
5428     return None;
5429 
5430   Offset = MI.getOperand(2).getImm() * Sign;
5431   return RegImmPair{MI.getOperand(1).getReg(), Offset};
5432 }
5433 
5434 bool llvm::registerDefinedBetween(unsigned Reg,
5435                                   MachineBasicBlock::iterator From,
5436                                   MachineBasicBlock::iterator To,
5437                                   const TargetRegisterInfo *TRI) {
5438   for (auto I = From; I != To; ++I)
5439     if (I->modifiesRegister(Reg, TRI))
5440       return true;
5441   return false;
5442 }
5443 
5444 MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5445                                          const TargetRegisterInfo *TRI) {
5446   // Search backwards to the instruction that defines CSPR. This may or not
5447   // be a CMP, we check that after this loop. If we find another instruction
5448   // that reads cpsr, we return nullptr.
5449   MachineBasicBlock::iterator CmpMI = Br;
5450   while (CmpMI != Br->getParent()->begin()) {
5451     --CmpMI;
5452     if (CmpMI->modifiesRegister(ARM::CPSR, TRI))
5453       break;
5454     if (CmpMI->readsRegister(ARM::CPSR, TRI))
5455       break;
5456   }
5457 
5458   // Check that this inst is a CMP r[0-7], #0 and that the register
5459   // is not redefined between the cmp and the br.
5460   if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5461     return nullptr;
5462   Register Reg = CmpMI->getOperand(0).getReg();
5463   Register PredReg;
5464   ARMCC::CondCodes Pred = getInstrPredicate(*CmpMI, PredReg);
5465   if (Pred != ARMCC::AL || CmpMI->getOperand(1).getImm() != 0)
5466     return nullptr;
5467   if (!isARMLowRegister(Reg))
5468     return nullptr;
5469   if (registerDefinedBetween(Reg, CmpMI->getNextNode(), Br, TRI))
5470     return nullptr;
5471 
5472   return &*CmpMI;
5473 }
5474 
5475 unsigned llvm::ConstantMaterializationCost(unsigned Val,
5476                                            const ARMSubtarget *Subtarget,
5477                                            bool ForCodesize) {
5478   if (Subtarget->isThumb()) {
5479     if (Val <= 255) // MOV
5480       return ForCodesize ? 2 : 1;
5481     if (Subtarget->hasV6T2Ops() && (Val <= 0xffff ||                    // MOV
5482                                     ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
5483                                     ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
5484       return ForCodesize ? 4 : 1;
5485     if (Val <= 510) // MOV + ADDi8
5486       return ForCodesize ? 4 : 2;
5487     if (~Val <= 255) // MOV + MVN
5488       return ForCodesize ? 4 : 2;
5489     if (ARM_AM::isThumbImmShiftedVal(Val)) // MOV + LSL
5490       return ForCodesize ? 4 : 2;
5491   } else {
5492     if (ARM_AM::getSOImmVal(Val) != -1) // MOV
5493       return ForCodesize ? 4 : 1;
5494     if (ARM_AM::getSOImmVal(~Val) != -1) // MVN
5495       return ForCodesize ? 4 : 1;
5496     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5497       return ForCodesize ? 4 : 1;
5498     if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs
5499       return ForCodesize ? 8 : 2;
5500   }
5501   if (Subtarget->useMovt()) // MOVW + MOVT
5502     return ForCodesize ? 8 : 2;
5503   return ForCodesize ? 8 : 3; // Literal pool load
5504 }
5505 
5506 bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5507                                                const ARMSubtarget *Subtarget,
5508                                                bool ForCodesize) {
5509   // Check with ForCodesize
5510   unsigned Cost1 = ConstantMaterializationCost(Val1, Subtarget, ForCodesize);
5511   unsigned Cost2 = ConstantMaterializationCost(Val2, Subtarget, ForCodesize);
5512   if (Cost1 < Cost2)
5513     return true;
5514   if (Cost1 > Cost2)
5515     return false;
5516 
5517   // If they are equal, try with !ForCodesize
5518   return ConstantMaterializationCost(Val1, Subtarget, !ForCodesize) <
5519          ConstantMaterializationCost(Val2, Subtarget, !ForCodesize);
5520 }
5521 
5522 /// Constants defining how certain sequences should be outlined.
5523 /// This encompasses how an outlined function should be called, and what kind of
5524 /// frame should be emitted for that outlined function.
5525 ///
5526 /// \p MachineOutlinerTailCall implies that the function is being created from
5527 /// a sequence of instructions ending in a return.
5528 ///
5529 /// That is,
5530 ///
5531 /// I1                                OUTLINED_FUNCTION:
5532 /// I2    --> B OUTLINED_FUNCTION     I1
5533 /// BX LR                             I2
5534 ///                                   BX LR
5535 ///
5536 /// +-------------------------+--------+-----+
5537 /// |                         | Thumb2 | ARM |
5538 /// +-------------------------+--------+-----+
5539 /// | Call overhead in Bytes  |      4 |   4 |
5540 /// | Frame overhead in Bytes |      0 |   0 |
5541 /// | Stack fixup required    |     No |  No |
5542 /// +-------------------------+--------+-----+
5543 ///
5544 /// \p MachineOutlinerThunk implies that the function is being created from
5545 /// a sequence of instructions ending in a call. The outlined function is
5546 /// called with a BL instruction, and the outlined function tail-calls the
5547 /// original call destination.
5548 ///
5549 /// That is,
5550 ///
5551 /// I1                                OUTLINED_FUNCTION:
5552 /// I2   --> BL OUTLINED_FUNCTION     I1
5553 /// BL f                              I2
5554 ///                                   B f
5555 ///
5556 /// +-------------------------+--------+-----+
5557 /// |                         | Thumb2 | ARM |
5558 /// +-------------------------+--------+-----+
5559 /// | Call overhead in Bytes  |      4 |   4 |
5560 /// | Frame overhead in Bytes |      0 |   0 |
5561 /// | Stack fixup required    |     No |  No |
5562 /// +-------------------------+--------+-----+
5563 
5564 enum MachineOutlinerClass { MachineOutlinerTailCall, MachineOutlinerThunk };
5565 
5566 enum MachineOutlinerMBBFlags {
5567   LRUnavailableSomewhere = 0x2,
5568   HasCalls = 0x4,
5569   UnsafeRegsDead = 0x8
5570 };
5571 
5572 struct OutlinerCosts {
5573   const int CallTailCall;
5574   const int FrameTailCall;
5575   const int CallThunk;
5576   const int FrameThunk;
5577 
5578   OutlinerCosts(const ARMSubtarget &target)
5579       : CallTailCall(target.isThumb() ? 4 : 4),
5580         FrameTailCall(target.isThumb() ? 0 : 0),
5581         CallThunk(target.isThumb() ? 4 : 4),
5582         FrameThunk(target.isThumb() ? 0 : 0) {}
5583 };
5584 
5585 outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
5586     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5587   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5588   unsigned SequenceSize =
5589       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5590                       [this](unsigned Sum, const MachineInstr &MI) {
5591                         return Sum + getInstSizeInBytes(MI);
5592                       });
5593 
5594   // Properties about candidate MBBs that hold for all of them.
5595   unsigned FlagsSetInAll = 0xF;
5596 
5597   // Compute liveness information for each candidate, and set FlagsSetInAll.
5598   const TargetRegisterInfo &TRI = getRegisterInfo();
5599   std::for_each(
5600       RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5601       [&FlagsSetInAll](outliner::Candidate &C) { FlagsSetInAll &= C.Flags; });
5602 
5603   // According to the ARM Procedure Call Standard, the following are
5604   // undefined on entry/exit from a function call:
5605   //
5606   // * Register R12(IP),
5607   // * Condition codes (and thus the CPSR register)
5608   //
5609   // Since we control the instructions which are part of the outlined regions
5610   // we don't need to be fully compliant with the AAPCS, but we have to
5611   // guarantee that if a veneer is inserted at link time the code is still
5612   // correct.  Because of this, we can't outline any sequence of instructions
5613   // where one of these registers is live into/across it. Thus, we need to
5614   // delete those candidates.
5615   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5616     // If the unsafe registers in this block are all dead, then we don't need
5617     // to compute liveness here.
5618     if (C.Flags & UnsafeRegsDead)
5619       return false;
5620     C.initLRU(TRI);
5621     LiveRegUnits LRU = C.LRU;
5622     return (!LRU.available(ARM::R12) || !LRU.available(ARM::CPSR));
5623   };
5624 
5625   // Are there any candidates where those registers are live?
5626   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5627     // Erase every candidate that violates the restrictions above. (It could be
5628     // true that we have viable candidates, so it's not worth bailing out in
5629     // the case that, say, 1 out of 20 candidates violate the restructions.)
5630     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5631                                               RepeatedSequenceLocs.end(),
5632                                               CantGuaranteeValueAcrossCall),
5633                                RepeatedSequenceLocs.end());
5634 
5635     // If the sequence doesn't have enough candidates left, then we're done.
5636     if (RepeatedSequenceLocs.size() < 2)
5637       return outliner::OutlinedFunction();
5638   }
5639 
5640   // At this point, we have only "safe" candidates to outline. Figure out
5641   // frame + call instruction information.
5642 
5643   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5644 
5645   // Helper lambda which sets call information for every candidate.
5646   auto SetCandidateCallInfo =
5647       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5648         for (outliner::Candidate &C : RepeatedSequenceLocs)
5649           C.setCallInfo(CallID, NumBytesForCall);
5650       };
5651 
5652   auto Costs = std::make_unique<OutlinerCosts>(Subtarget);
5653   unsigned FrameID = 0;
5654   unsigned NumBytesToCreateFrame = 0;
5655 
5656   // If the last instruction in any candidate is a terminator, then we should
5657   // tail call all of the candidates.
5658   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
5659     FrameID = MachineOutlinerTailCall;
5660     NumBytesToCreateFrame = Costs->FrameTailCall;
5661     SetCandidateCallInfo(MachineOutlinerTailCall, Costs->CallTailCall);
5662   } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX ||
5663              LastInstrOpcode == ARM::tBL || LastInstrOpcode == ARM::tBLXr ||
5664              LastInstrOpcode == ARM::tBLXi) {
5665     FrameID = MachineOutlinerThunk;
5666     NumBytesToCreateFrame = Costs->FrameThunk;
5667     SetCandidateCallInfo(MachineOutlinerThunk, Costs->CallThunk);
5668   } else
5669     return outliner::OutlinedFunction();
5670 
5671   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
5672                                     NumBytesToCreateFrame, FrameID);
5673 }
5674 
5675 bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
5676     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
5677   const Function &F = MF.getFunction();
5678 
5679   // Can F be deduplicated by the linker? If it can, don't outline from it.
5680   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
5681     return false;
5682 
5683   // Don't outline from functions with section markings; the program could
5684   // expect that all the code is in the named section.
5685   // FIXME: Allow outlining from multiple functions with the same section
5686   // marking.
5687   if (F.hasSection())
5688     return false;
5689 
5690   // FIXME: Thumb1 outlining is not handled
5691   if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
5692     return false;
5693 
5694   // It's safe to outline from MF.
5695   return true;
5696 }
5697 
5698 bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
5699                                               unsigned &Flags) const {
5700   // Check if LR is available through all of the MBB. If it's not, then set
5701   // a flag.
5702   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
5703          "Suitable Machine Function for outlining must track liveness");
5704 
5705   LiveRegUnits LRU(getRegisterInfo());
5706 
5707   std::for_each(MBB.rbegin(), MBB.rend(),
5708                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
5709 
5710   // Check if each of the unsafe registers are available...
5711   bool R12AvailableInBlock = LRU.available(ARM::R12);
5712   bool CPSRAvailableInBlock = LRU.available(ARM::CPSR);
5713 
5714   // If all of these are dead (and not live out), we know we don't have to check
5715   // them later.
5716   if (R12AvailableInBlock && CPSRAvailableInBlock)
5717     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
5718 
5719   // Now, add the live outs to the set.
5720   LRU.addLiveOuts(MBB);
5721 
5722   // If any of these registers is available in the MBB, but also a live out of
5723   // the block, then we know outlining is unsafe.
5724   if (R12AvailableInBlock && !LRU.available(ARM::R12))
5725     return false;
5726   if (CPSRAvailableInBlock && !LRU.available(ARM::CPSR))
5727     return false;
5728 
5729   // Check if there's a call inside this MachineBasicBlock.  If there is, then
5730   // set a flag.
5731   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
5732     Flags |= MachineOutlinerMBBFlags::HasCalls;
5733 
5734   if (!LRU.available(ARM::LR))
5735     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
5736 
5737   return true;
5738 }
5739 
5740 outliner::InstrType
5741 ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
5742                                    unsigned Flags) const {
5743   MachineInstr &MI = *MIT;
5744   const TargetRegisterInfo *TRI = &getRegisterInfo();
5745 
5746   // Be conservative with inline ASM
5747   if (MI.isInlineAsm())
5748     return outliner::InstrType::Illegal;
5749 
5750   // Don't allow debug values to impact outlining type.
5751   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
5752     return outliner::InstrType::Invisible;
5753 
5754   // At this point, KILL or IMPLICIT_DEF instructions don't really tell us much
5755   // so we can go ahead and skip over them.
5756   if (MI.isKill() || MI.isImplicitDef())
5757     return outliner::InstrType::Invisible;
5758 
5759   // PIC instructions contain labels, outlining them would break offset
5760   // computing.  unsigned Opc = MI.getOpcode();
5761   unsigned Opc = MI.getOpcode();
5762   if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
5763       Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
5764       Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
5765       Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
5766       Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
5767       Opc == ARM::t2MOV_ga_pcrel)
5768     return outliner::InstrType::Illegal;
5769 
5770   // Be conservative with ARMv8.1 MVE instructions.
5771   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
5772       Opc == ARM::t2WhileLoopStart || Opc == ARM::t2LoopDec ||
5773       Opc == ARM::t2LoopEnd)
5774     return outliner::InstrType::Illegal;
5775 
5776   const MCInstrDesc &MCID = MI.getDesc();
5777   uint64_t MIFlags = MCID.TSFlags;
5778   if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
5779     return outliner::InstrType::Illegal;
5780 
5781   // Is this a terminator for a basic block?
5782   if (MI.isTerminator()) {
5783     // Don't outline if the branch is not unconditional.
5784     if (isPredicated(MI))
5785       return outliner::InstrType::Illegal;
5786 
5787     // Is this the end of a function?
5788     if (MI.getParent()->succ_empty())
5789       return outliner::InstrType::Legal;
5790 
5791     // It's not, so don't outline it.
5792     return outliner::InstrType::Illegal;
5793   }
5794 
5795   // Make sure none of the operands are un-outlinable.
5796   for (const MachineOperand &MOP : MI.operands()) {
5797     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
5798         MOP.isTargetIndex())
5799       return outliner::InstrType::Illegal;
5800   }
5801 
5802   // Don't outline if link register or program counter value are used.
5803   if (MI.readsRegister(ARM::LR, TRI) || MI.readsRegister(ARM::PC, TRI))
5804     return outliner::InstrType::Illegal;
5805 
5806   if (MI.isCall()) {
5807     // If we don't know anything about the callee, assume it depends on the
5808     // stack layout of the caller. In that case, it's only legal to outline
5809     // as a tail-call.  Whitelist the call instructions we know about so we
5810     // don't get unexpected results with call pseudo-instructions.
5811     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
5812     if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX ||
5813         Opc == ARM::tBLXr || Opc == ARM::tBLXi)
5814       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
5815 
5816     return UnknownCallOutlineType;
5817   }
5818 
5819   // Since calls are handled, don't touch LR or PC
5820   if (MI.modifiesRegister(ARM::LR, TRI) || MI.modifiesRegister(ARM::PC, TRI))
5821     return outliner::InstrType::Illegal;
5822 
5823   // Be conservative with IT blocks.
5824   if (MI.readsRegister(ARM::ITSTATE, TRI) ||
5825       MI.modifiesRegister(ARM::ITSTATE, TRI))
5826     return outliner::InstrType::Illegal;
5827 
5828   // Don't outline positions.
5829   if (MI.isPosition())
5830     return outliner::InstrType::Illegal;
5831 
5832   return outliner::InstrType::Legal;
5833 }
5834 
5835 void ARMBaseInstrInfo::buildOutlinedFrame(
5836     MachineBasicBlock &MBB, MachineFunction &MF,
5837     const outliner::OutlinedFunction &OF) const {
5838   // For thunk outlining, rewrite the last instruction from a call to a
5839   // tail-call.
5840   if (OF.FrameConstructionID == MachineOutlinerThunk) {
5841     MachineInstr *Call = &*--MBB.instr_end();
5842     bool isThumb = Subtarget.isThumb();
5843     unsigned FuncOp = isThumb ? 2 : 0;
5844     unsigned Opc = Call->getOperand(FuncOp).isReg()
5845                        ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
5846                        : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
5847                                                              : ARM::tTAILJMPdND
5848                                  : ARM::TAILJMPd;
5849     MachineInstrBuilder MIB = BuildMI(MBB, MBB.end(), DebugLoc(), get(Opc))
5850                                   .add(Call->getOperand(FuncOp));
5851     if (isThumb && !Call->getOperand(FuncOp).isReg())
5852       MIB.add(predOps(ARMCC::AL));
5853     Call->eraseFromParent();
5854   }
5855 }
5856 
5857 MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
5858     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
5859     MachineFunction &MF, const outliner::Candidate &C) const {
5860   MachineInstrBuilder MIB;
5861   MachineBasicBlock::iterator CallPt;
5862   unsigned Opc;
5863   bool isThumb = Subtarget.isThumb();
5864 
5865   // Are we tail calling?
5866   if (C.CallConstructionID == MachineOutlinerTailCall) {
5867     // If yes, then we can just branch to the label.
5868     Opc = isThumb
5869               ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
5870               : ARM::TAILJMPd;
5871     MIB = BuildMI(MF, DebugLoc(), get(Opc))
5872               .addGlobalAddress(M.getNamedValue(MF.getName()));
5873     if (isThumb)
5874       MIB.add(predOps(ARMCC::AL));
5875     It = MBB.insert(It, MIB);
5876     return It;
5877   }
5878 
5879   // Create the call instruction.
5880   Opc = isThumb ? ARM::tBL : ARM::BL;
5881   MachineInstrBuilder CallMIB = BuildMI(MF, DebugLoc(), get(Opc));
5882   if (isThumb)
5883     CallMIB.add(predOps(ARMCC::AL));
5884   CallMIB.addGlobalAddress(M.getNamedValue(MF.getName()));
5885 
5886   // Insert the call.
5887   It = MBB.insert(It, CallMIB);
5888   return It;
5889 }
5890