1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64InstrInfo.h"
14 #include "AArch64MachineFunctionInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "Utils/AArch64BaseInfo.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineModuleInfo.h"
28 #include "llvm/CodeGen/MachineOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetRegisterInfo.h"
32 #include "llvm/CodeGen/TargetSubtargetInfo.h"
33 #include "llvm/IR/DebugInfoMetadata.h"
34 #include "llvm/IR/DebugLoc.h"
35 #include "llvm/IR/GlobalValue.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/MCInstrDesc.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Compiler.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <cassert>
48 #include <cstdint>
49 #include <iterator>
50 #include <utility>
51 
52 using namespace llvm;
53 
54 #define GET_INSTRINFO_CTOR_DTOR
55 #include "AArch64GenInstrInfo.inc"
56 
57 static cl::opt<unsigned> TBZDisplacementBits(
58     "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
59     cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
60 
61 static cl::opt<unsigned> CBZDisplacementBits(
62     "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
63     cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
64 
65 static cl::opt<unsigned>
66     BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
67                         cl::desc("Restrict range of Bcc instructions (DEBUG)"));
68 
69 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
70     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
71                           AArch64::CATCHRET),
72       RI(STI.getTargetTriple()), Subtarget(STI) {}
73 
74 /// GetInstSize - Return the number of bytes of code the specified
75 /// instruction may be.  This returns the maximum number of bytes.
76 unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
77   const MachineBasicBlock &MBB = *MI.getParent();
78   const MachineFunction *MF = MBB.getParent();
79   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
80 
81   {
82     auto Op = MI.getOpcode();
83     if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
84       return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
85   }
86 
87   // Meta-instructions emit no code.
88   if (MI.isMetaInstruction())
89     return 0;
90 
91   // FIXME: We currently only handle pseudoinstructions that don't get expanded
92   //        before the assembly printer.
93   unsigned NumBytes = 0;
94   const MCInstrDesc &Desc = MI.getDesc();
95   switch (Desc.getOpcode()) {
96   default:
97     // Anything not explicitly designated otherwise is a normal 4-byte insn.
98     NumBytes = 4;
99     break;
100   case TargetOpcode::STACKMAP:
101     // The upper bound for a stackmap intrinsic is the full length of its shadow
102     NumBytes = StackMapOpers(&MI).getNumPatchBytes();
103     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
104     break;
105   case TargetOpcode::PATCHPOINT:
106     // The size of the patchpoint intrinsic is the number of bytes requested
107     NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
108     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
109     break;
110   case TargetOpcode::STATEPOINT:
111     NumBytes = StatepointOpers(&MI).getNumPatchBytes();
112     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
113     // No patch bytes means a normal call inst is emitted
114     if (NumBytes == 0)
115       NumBytes = 4;
116     break;
117   case AArch64::TLSDESC_CALLSEQ:
118     // This gets lowered to an instruction sequence which takes 16 bytes
119     NumBytes = 16;
120     break;
121   case AArch64::SpeculationBarrierISBDSBEndBB:
122     // This gets lowered to 2 4-byte instructions.
123     NumBytes = 8;
124     break;
125   case AArch64::SpeculationBarrierSBEndBB:
126     // This gets lowered to 1 4-byte instructions.
127     NumBytes = 4;
128     break;
129   case AArch64::JumpTableDest32:
130   case AArch64::JumpTableDest16:
131   case AArch64::JumpTableDest8:
132     NumBytes = 12;
133     break;
134   case AArch64::SPACE:
135     NumBytes = MI.getOperand(1).getImm();
136     break;
137   case TargetOpcode::BUNDLE:
138     NumBytes = getInstBundleLength(MI);
139     break;
140   }
141 
142   return NumBytes;
143 }
144 
145 unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
146   unsigned Size = 0;
147   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
148   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
149   while (++I != E && I->isInsideBundle()) {
150     assert(!I->isBundle() && "No nested bundle!");
151     Size += getInstSizeInBytes(*I);
152   }
153   return Size;
154 }
155 
156 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
157                             SmallVectorImpl<MachineOperand> &Cond) {
158   // Block ends with fall-through condbranch.
159   switch (LastInst->getOpcode()) {
160   default:
161     llvm_unreachable("Unknown branch instruction?");
162   case AArch64::Bcc:
163     Target = LastInst->getOperand(1).getMBB();
164     Cond.push_back(LastInst->getOperand(0));
165     break;
166   case AArch64::CBZW:
167   case AArch64::CBZX:
168   case AArch64::CBNZW:
169   case AArch64::CBNZX:
170     Target = LastInst->getOperand(1).getMBB();
171     Cond.push_back(MachineOperand::CreateImm(-1));
172     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
173     Cond.push_back(LastInst->getOperand(0));
174     break;
175   case AArch64::TBZW:
176   case AArch64::TBZX:
177   case AArch64::TBNZW:
178   case AArch64::TBNZX:
179     Target = LastInst->getOperand(2).getMBB();
180     Cond.push_back(MachineOperand::CreateImm(-1));
181     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
182     Cond.push_back(LastInst->getOperand(0));
183     Cond.push_back(LastInst->getOperand(1));
184   }
185 }
186 
187 static unsigned getBranchDisplacementBits(unsigned Opc) {
188   switch (Opc) {
189   default:
190     llvm_unreachable("unexpected opcode!");
191   case AArch64::B:
192     return 64;
193   case AArch64::TBNZW:
194   case AArch64::TBZW:
195   case AArch64::TBNZX:
196   case AArch64::TBZX:
197     return TBZDisplacementBits;
198   case AArch64::CBNZW:
199   case AArch64::CBZW:
200   case AArch64::CBNZX:
201   case AArch64::CBZX:
202     return CBZDisplacementBits;
203   case AArch64::Bcc:
204     return BCCDisplacementBits;
205   }
206 }
207 
208 bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
209                                              int64_t BrOffset) const {
210   unsigned Bits = getBranchDisplacementBits(BranchOp);
211   assert(Bits >= 3 && "max branch displacement must be enough to jump"
212                       "over conditional branch expansion");
213   return isIntN(Bits, BrOffset / 4);
214 }
215 
216 MachineBasicBlock *
217 AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
218   switch (MI.getOpcode()) {
219   default:
220     llvm_unreachable("unexpected opcode!");
221   case AArch64::B:
222     return MI.getOperand(0).getMBB();
223   case AArch64::TBZW:
224   case AArch64::TBNZW:
225   case AArch64::TBZX:
226   case AArch64::TBNZX:
227     return MI.getOperand(2).getMBB();
228   case AArch64::CBZW:
229   case AArch64::CBNZW:
230   case AArch64::CBZX:
231   case AArch64::CBNZX:
232   case AArch64::Bcc:
233     return MI.getOperand(1).getMBB();
234   }
235 }
236 
237 // Branch analysis.
238 bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
239                                      MachineBasicBlock *&TBB,
240                                      MachineBasicBlock *&FBB,
241                                      SmallVectorImpl<MachineOperand> &Cond,
242                                      bool AllowModify) const {
243   // If the block has no terminators, it just falls into the block after it.
244   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
245   if (I == MBB.end())
246     return false;
247 
248   // Skip over SpeculationBarrierEndBB terminators
249   if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
250       I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
251     --I;
252   }
253 
254   if (!isUnpredicatedTerminator(*I))
255     return false;
256 
257   // Get the last instruction in the block.
258   MachineInstr *LastInst = &*I;
259 
260   // If there is only one terminator instruction, process it.
261   unsigned LastOpc = LastInst->getOpcode();
262   if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
263     if (isUncondBranchOpcode(LastOpc)) {
264       TBB = LastInst->getOperand(0).getMBB();
265       return false;
266     }
267     if (isCondBranchOpcode(LastOpc)) {
268       // Block ends with fall-through condbranch.
269       parseCondBranch(LastInst, TBB, Cond);
270       return false;
271     }
272     return true; // Can't handle indirect branch.
273   }
274 
275   // Get the instruction before it if it is a terminator.
276   MachineInstr *SecondLastInst = &*I;
277   unsigned SecondLastOpc = SecondLastInst->getOpcode();
278 
279   // If AllowModify is true and the block ends with two or more unconditional
280   // branches, delete all but the first unconditional branch.
281   if (AllowModify && isUncondBranchOpcode(LastOpc)) {
282     while (isUncondBranchOpcode(SecondLastOpc)) {
283       LastInst->eraseFromParent();
284       LastInst = SecondLastInst;
285       LastOpc = LastInst->getOpcode();
286       if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
287         // Return now the only terminator is an unconditional branch.
288         TBB = LastInst->getOperand(0).getMBB();
289         return false;
290       } else {
291         SecondLastInst = &*I;
292         SecondLastOpc = SecondLastInst->getOpcode();
293       }
294     }
295   }
296 
297   // If we're allowed to modify and the block ends in a unconditional branch
298   // which could simply fallthrough, remove the branch.  (Note: This case only
299   // matters when we can't understand the whole sequence, otherwise it's also
300   // handled by BranchFolding.cpp.)
301   if (AllowModify && isUncondBranchOpcode(LastOpc) &&
302       MBB.isLayoutSuccessor(getBranchDestBlock(*LastInst))) {
303     LastInst->eraseFromParent();
304     LastInst = SecondLastInst;
305     LastOpc = LastInst->getOpcode();
306     if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
307       assert(!isUncondBranchOpcode(LastOpc) &&
308              "unreachable unconditional branches removed above");
309 
310       if (isCondBranchOpcode(LastOpc)) {
311         // Block ends with fall-through condbranch.
312         parseCondBranch(LastInst, TBB, Cond);
313         return false;
314       }
315       return true; // Can't handle indirect branch.
316     } else {
317       SecondLastInst = &*I;
318       SecondLastOpc = SecondLastInst->getOpcode();
319     }
320   }
321 
322   // If there are three terminators, we don't know what sort of block this is.
323   if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
324     return true;
325 
326   // If the block ends with a B and a Bcc, handle it.
327   if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
328     parseCondBranch(SecondLastInst, TBB, Cond);
329     FBB = LastInst->getOperand(0).getMBB();
330     return false;
331   }
332 
333   // If the block ends with two unconditional branches, handle it.  The second
334   // one is not executed, so remove it.
335   if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
336     TBB = SecondLastInst->getOperand(0).getMBB();
337     I = LastInst;
338     if (AllowModify)
339       I->eraseFromParent();
340     return false;
341   }
342 
343   // ...likewise if it ends with an indirect branch followed by an unconditional
344   // branch.
345   if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
346     I = LastInst;
347     if (AllowModify)
348       I->eraseFromParent();
349     return true;
350   }
351 
352   // Otherwise, can't handle this.
353   return true;
354 }
355 
356 bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
357                                               MachineBranchPredicate &MBP,
358                                               bool AllowModify) const {
359   // For the moment, handle only a block which ends with a cb(n)zx followed by
360   // a fallthrough.  Why this?  Because it is a common form.
361   // TODO: Should we handle b.cc?
362 
363   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
364   if (I == MBB.end())
365     return true;
366 
367   // Skip over SpeculationBarrierEndBB terminators
368   if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
369       I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
370     --I;
371   }
372 
373   if (!isUnpredicatedTerminator(*I))
374     return true;
375 
376   // Get the last instruction in the block.
377   MachineInstr *LastInst = &*I;
378   unsigned LastOpc = LastInst->getOpcode();
379   if (!isCondBranchOpcode(LastOpc))
380     return true;
381 
382   switch (LastOpc) {
383   default:
384     return true;
385   case AArch64::CBZW:
386   case AArch64::CBZX:
387   case AArch64::CBNZW:
388   case AArch64::CBNZX:
389     break;
390   };
391 
392   MBP.TrueDest = LastInst->getOperand(1).getMBB();
393   assert(MBP.TrueDest && "expected!");
394   MBP.FalseDest = MBB.getNextNode();
395 
396   MBP.ConditionDef = nullptr;
397   MBP.SingleUseCondition = false;
398 
399   MBP.LHS = LastInst->getOperand(0);
400   MBP.RHS = MachineOperand::CreateImm(0);
401   MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
402                                             : MachineBranchPredicate::PRED_EQ;
403   return false;
404 }
405 
406 bool AArch64InstrInfo::reverseBranchCondition(
407     SmallVectorImpl<MachineOperand> &Cond) const {
408   if (Cond[0].getImm() != -1) {
409     // Regular Bcc
410     AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
411     Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
412   } else {
413     // Folded compare-and-branch
414     switch (Cond[1].getImm()) {
415     default:
416       llvm_unreachable("Unknown conditional branch!");
417     case AArch64::CBZW:
418       Cond[1].setImm(AArch64::CBNZW);
419       break;
420     case AArch64::CBNZW:
421       Cond[1].setImm(AArch64::CBZW);
422       break;
423     case AArch64::CBZX:
424       Cond[1].setImm(AArch64::CBNZX);
425       break;
426     case AArch64::CBNZX:
427       Cond[1].setImm(AArch64::CBZX);
428       break;
429     case AArch64::TBZW:
430       Cond[1].setImm(AArch64::TBNZW);
431       break;
432     case AArch64::TBNZW:
433       Cond[1].setImm(AArch64::TBZW);
434       break;
435     case AArch64::TBZX:
436       Cond[1].setImm(AArch64::TBNZX);
437       break;
438     case AArch64::TBNZX:
439       Cond[1].setImm(AArch64::TBZX);
440       break;
441     }
442   }
443 
444   return false;
445 }
446 
447 unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
448                                         int *BytesRemoved) const {
449   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
450   if (I == MBB.end())
451     return 0;
452 
453   if (!isUncondBranchOpcode(I->getOpcode()) &&
454       !isCondBranchOpcode(I->getOpcode()))
455     return 0;
456 
457   // Remove the branch.
458   I->eraseFromParent();
459 
460   I = MBB.end();
461 
462   if (I == MBB.begin()) {
463     if (BytesRemoved)
464       *BytesRemoved = 4;
465     return 1;
466   }
467   --I;
468   if (!isCondBranchOpcode(I->getOpcode())) {
469     if (BytesRemoved)
470       *BytesRemoved = 4;
471     return 1;
472   }
473 
474   // Remove the branch.
475   I->eraseFromParent();
476   if (BytesRemoved)
477     *BytesRemoved = 8;
478 
479   return 2;
480 }
481 
482 void AArch64InstrInfo::instantiateCondBranch(
483     MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
484     ArrayRef<MachineOperand> Cond) const {
485   if (Cond[0].getImm() != -1) {
486     // Regular Bcc
487     BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
488   } else {
489     // Folded compare-and-branch
490     // Note that we use addOperand instead of addReg to keep the flags.
491     const MachineInstrBuilder MIB =
492         BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
493     if (Cond.size() > 3)
494       MIB.addImm(Cond[3].getImm());
495     MIB.addMBB(TBB);
496   }
497 }
498 
499 unsigned AArch64InstrInfo::insertBranch(
500     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
501     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
502   // Shouldn't be a fall through.
503   assert(TBB && "insertBranch must not be told to insert a fallthrough");
504 
505   if (!FBB) {
506     if (Cond.empty()) // Unconditional branch?
507       BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
508     else
509       instantiateCondBranch(MBB, DL, TBB, Cond);
510 
511     if (BytesAdded)
512       *BytesAdded = 4;
513 
514     return 1;
515   }
516 
517   // Two-way conditional branch.
518   instantiateCondBranch(MBB, DL, TBB, Cond);
519   BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
520 
521   if (BytesAdded)
522     *BytesAdded = 8;
523 
524   return 2;
525 }
526 
527 // Find the original register that VReg is copied from.
528 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
529   while (Register::isVirtualRegister(VReg)) {
530     const MachineInstr *DefMI = MRI.getVRegDef(VReg);
531     if (!DefMI->isFullCopy())
532       return VReg;
533     VReg = DefMI->getOperand(1).getReg();
534   }
535   return VReg;
536 }
537 
538 // Determine if VReg is defined by an instruction that can be folded into a
539 // csel instruction. If so, return the folded opcode, and the replacement
540 // register.
541 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
542                                 unsigned *NewVReg = nullptr) {
543   VReg = removeCopies(MRI, VReg);
544   if (!Register::isVirtualRegister(VReg))
545     return 0;
546 
547   bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
548   const MachineInstr *DefMI = MRI.getVRegDef(VReg);
549   unsigned Opc = 0;
550   unsigned SrcOpNum = 0;
551   switch (DefMI->getOpcode()) {
552   case AArch64::ADDSXri:
553   case AArch64::ADDSWri:
554     // if NZCV is used, do not fold.
555     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
556       return 0;
557     // fall-through to ADDXri and ADDWri.
558     LLVM_FALLTHROUGH;
559   case AArch64::ADDXri:
560   case AArch64::ADDWri:
561     // add x, 1 -> csinc.
562     if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
563         DefMI->getOperand(3).getImm() != 0)
564       return 0;
565     SrcOpNum = 1;
566     Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
567     break;
568 
569   case AArch64::ORNXrr:
570   case AArch64::ORNWrr: {
571     // not x -> csinv, represented as orn dst, xzr, src.
572     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
573     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
574       return 0;
575     SrcOpNum = 2;
576     Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
577     break;
578   }
579 
580   case AArch64::SUBSXrr:
581   case AArch64::SUBSWrr:
582     // if NZCV is used, do not fold.
583     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
584       return 0;
585     // fall-through to SUBXrr and SUBWrr.
586     LLVM_FALLTHROUGH;
587   case AArch64::SUBXrr:
588   case AArch64::SUBWrr: {
589     // neg x -> csneg, represented as sub dst, xzr, src.
590     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
591     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
592       return 0;
593     SrcOpNum = 2;
594     Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
595     break;
596   }
597   default:
598     return 0;
599   }
600   assert(Opc && SrcOpNum && "Missing parameters");
601 
602   if (NewVReg)
603     *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
604   return Opc;
605 }
606 
607 bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
608                                        ArrayRef<MachineOperand> Cond,
609                                        Register DstReg, Register TrueReg,
610                                        Register FalseReg, int &CondCycles,
611                                        int &TrueCycles,
612                                        int &FalseCycles) const {
613   // Check register classes.
614   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
615   const TargetRegisterClass *RC =
616       RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
617   if (!RC)
618     return false;
619 
620   // Also need to check the dest regclass, in case we're trying to optimize
621   // something like:
622   // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
623   if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
624     return false;
625 
626   // Expanding cbz/tbz requires an extra cycle of latency on the condition.
627   unsigned ExtraCondLat = Cond.size() != 1;
628 
629   // GPRs are handled by csel.
630   // FIXME: Fold in x+1, -x, and ~x when applicable.
631   if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
632       AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
633     // Single-cycle csel, csinc, csinv, and csneg.
634     CondCycles = 1 + ExtraCondLat;
635     TrueCycles = FalseCycles = 1;
636     if (canFoldIntoCSel(MRI, TrueReg))
637       TrueCycles = 0;
638     else if (canFoldIntoCSel(MRI, FalseReg))
639       FalseCycles = 0;
640     return true;
641   }
642 
643   // Scalar floating point is handled by fcsel.
644   // FIXME: Form fabs, fmin, and fmax when applicable.
645   if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
646       AArch64::FPR32RegClass.hasSubClassEq(RC)) {
647     CondCycles = 5 + ExtraCondLat;
648     TrueCycles = FalseCycles = 2;
649     return true;
650   }
651 
652   // Can't do vectors.
653   return false;
654 }
655 
656 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
657                                     MachineBasicBlock::iterator I,
658                                     const DebugLoc &DL, Register DstReg,
659                                     ArrayRef<MachineOperand> Cond,
660                                     Register TrueReg, Register FalseReg) const {
661   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
662 
663   // Parse the condition code, see parseCondBranch() above.
664   AArch64CC::CondCode CC;
665   switch (Cond.size()) {
666   default:
667     llvm_unreachable("Unknown condition opcode in Cond");
668   case 1: // b.cc
669     CC = AArch64CC::CondCode(Cond[0].getImm());
670     break;
671   case 3: { // cbz/cbnz
672     // We must insert a compare against 0.
673     bool Is64Bit;
674     switch (Cond[1].getImm()) {
675     default:
676       llvm_unreachable("Unknown branch opcode in Cond");
677     case AArch64::CBZW:
678       Is64Bit = false;
679       CC = AArch64CC::EQ;
680       break;
681     case AArch64::CBZX:
682       Is64Bit = true;
683       CC = AArch64CC::EQ;
684       break;
685     case AArch64::CBNZW:
686       Is64Bit = false;
687       CC = AArch64CC::NE;
688       break;
689     case AArch64::CBNZX:
690       Is64Bit = true;
691       CC = AArch64CC::NE;
692       break;
693     }
694     Register SrcReg = Cond[2].getReg();
695     if (Is64Bit) {
696       // cmp reg, #0 is actually subs xzr, reg, #0.
697       MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
698       BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
699           .addReg(SrcReg)
700           .addImm(0)
701           .addImm(0);
702     } else {
703       MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
704       BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
705           .addReg(SrcReg)
706           .addImm(0)
707           .addImm(0);
708     }
709     break;
710   }
711   case 4: { // tbz/tbnz
712     // We must insert a tst instruction.
713     switch (Cond[1].getImm()) {
714     default:
715       llvm_unreachable("Unknown branch opcode in Cond");
716     case AArch64::TBZW:
717     case AArch64::TBZX:
718       CC = AArch64CC::EQ;
719       break;
720     case AArch64::TBNZW:
721     case AArch64::TBNZX:
722       CC = AArch64CC::NE;
723       break;
724     }
725     // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
726     if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
727       BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
728           .addReg(Cond[2].getReg())
729           .addImm(
730               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
731     else
732       BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
733           .addReg(Cond[2].getReg())
734           .addImm(
735               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
736     break;
737   }
738   }
739 
740   unsigned Opc = 0;
741   const TargetRegisterClass *RC = nullptr;
742   bool TryFold = false;
743   if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
744     RC = &AArch64::GPR64RegClass;
745     Opc = AArch64::CSELXr;
746     TryFold = true;
747   } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
748     RC = &AArch64::GPR32RegClass;
749     Opc = AArch64::CSELWr;
750     TryFold = true;
751   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
752     RC = &AArch64::FPR64RegClass;
753     Opc = AArch64::FCSELDrrr;
754   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
755     RC = &AArch64::FPR32RegClass;
756     Opc = AArch64::FCSELSrrr;
757   }
758   assert(RC && "Unsupported regclass");
759 
760   // Try folding simple instructions into the csel.
761   if (TryFold) {
762     unsigned NewVReg = 0;
763     unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
764     if (FoldedOpc) {
765       // The folded opcodes csinc, csinc and csneg apply the operation to
766       // FalseReg, so we need to invert the condition.
767       CC = AArch64CC::getInvertedCondCode(CC);
768       TrueReg = FalseReg;
769     } else
770       FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
771 
772     // Fold the operation. Leave any dead instructions for DCE to clean up.
773     if (FoldedOpc) {
774       FalseReg = NewVReg;
775       Opc = FoldedOpc;
776       // The extends the live range of NewVReg.
777       MRI.clearKillFlags(NewVReg);
778     }
779   }
780 
781   // Pull all virtual register into the appropriate class.
782   MRI.constrainRegClass(TrueReg, RC);
783   MRI.constrainRegClass(FalseReg, RC);
784 
785   // Insert the csel.
786   BuildMI(MBB, I, DL, get(Opc), DstReg)
787       .addReg(TrueReg)
788       .addReg(FalseReg)
789       .addImm(CC);
790 }
791 
792 /// Returns true if a MOVi32imm or MOVi64imm can be expanded to an  ORRxx.
793 static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
794   uint64_t Imm = MI.getOperand(1).getImm();
795   uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
796   uint64_t Encoding;
797   return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
798 }
799 
800 // FIXME: this implementation should be micro-architecture dependent, so a
801 // micro-architecture target hook should be introduced here in future.
802 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
803   if (!Subtarget.hasCustomCheapAsMoveHandling())
804     return MI.isAsCheapAsAMove();
805 
806   const unsigned Opcode = MI.getOpcode();
807 
808   // Firstly, check cases gated by features.
809 
810   if (Subtarget.hasZeroCycleZeroingFP()) {
811     if (Opcode == AArch64::FMOVH0 ||
812         Opcode == AArch64::FMOVS0 ||
813         Opcode == AArch64::FMOVD0)
814       return true;
815   }
816 
817   if (Subtarget.hasZeroCycleZeroingGP()) {
818     if (Opcode == TargetOpcode::COPY &&
819         (MI.getOperand(1).getReg() == AArch64::WZR ||
820          MI.getOperand(1).getReg() == AArch64::XZR))
821       return true;
822   }
823 
824   // Secondly, check cases specific to sub-targets.
825 
826   if (Subtarget.hasExynosCheapAsMoveHandling()) {
827     if (isExynosCheapAsMove(MI))
828       return true;
829 
830     return MI.isAsCheapAsAMove();
831   }
832 
833   // Finally, check generic cases.
834 
835   switch (Opcode) {
836   default:
837     return false;
838 
839   // add/sub on register without shift
840   case AArch64::ADDWri:
841   case AArch64::ADDXri:
842   case AArch64::SUBWri:
843   case AArch64::SUBXri:
844     return (MI.getOperand(3).getImm() == 0);
845 
846   // logical ops on immediate
847   case AArch64::ANDWri:
848   case AArch64::ANDXri:
849   case AArch64::EORWri:
850   case AArch64::EORXri:
851   case AArch64::ORRWri:
852   case AArch64::ORRXri:
853     return true;
854 
855   // logical ops on register without shift
856   case AArch64::ANDWrr:
857   case AArch64::ANDXrr:
858   case AArch64::BICWrr:
859   case AArch64::BICXrr:
860   case AArch64::EONWrr:
861   case AArch64::EONXrr:
862   case AArch64::EORWrr:
863   case AArch64::EORXrr:
864   case AArch64::ORNWrr:
865   case AArch64::ORNXrr:
866   case AArch64::ORRWrr:
867   case AArch64::ORRXrr:
868     return true;
869 
870   // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
871   // ORRXri, it is as cheap as MOV
872   case AArch64::MOVi32imm:
873     return canBeExpandedToORR(MI, 32);
874   case AArch64::MOVi64imm:
875     return canBeExpandedToORR(MI, 64);
876   }
877 
878   llvm_unreachable("Unknown opcode to check as cheap as a move!");
879 }
880 
881 bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
882   switch (MI.getOpcode()) {
883   default:
884     return false;
885 
886   case AArch64::ADDWrs:
887   case AArch64::ADDXrs:
888   case AArch64::ADDSWrs:
889   case AArch64::ADDSXrs: {
890     unsigned Imm = MI.getOperand(3).getImm();
891     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
892     if (ShiftVal == 0)
893       return true;
894     return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
895   }
896 
897   case AArch64::ADDWrx:
898   case AArch64::ADDXrx:
899   case AArch64::ADDXrx64:
900   case AArch64::ADDSWrx:
901   case AArch64::ADDSXrx:
902   case AArch64::ADDSXrx64: {
903     unsigned Imm = MI.getOperand(3).getImm();
904     switch (AArch64_AM::getArithExtendType(Imm)) {
905     default:
906       return false;
907     case AArch64_AM::UXTB:
908     case AArch64_AM::UXTH:
909     case AArch64_AM::UXTW:
910     case AArch64_AM::UXTX:
911       return AArch64_AM::getArithShiftValue(Imm) <= 4;
912     }
913   }
914 
915   case AArch64::SUBWrs:
916   case AArch64::SUBSWrs: {
917     unsigned Imm = MI.getOperand(3).getImm();
918     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
919     return ShiftVal == 0 ||
920            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
921   }
922 
923   case AArch64::SUBXrs:
924   case AArch64::SUBSXrs: {
925     unsigned Imm = MI.getOperand(3).getImm();
926     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
927     return ShiftVal == 0 ||
928            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
929   }
930 
931   case AArch64::SUBWrx:
932   case AArch64::SUBXrx:
933   case AArch64::SUBXrx64:
934   case AArch64::SUBSWrx:
935   case AArch64::SUBSXrx:
936   case AArch64::SUBSXrx64: {
937     unsigned Imm = MI.getOperand(3).getImm();
938     switch (AArch64_AM::getArithExtendType(Imm)) {
939     default:
940       return false;
941     case AArch64_AM::UXTB:
942     case AArch64_AM::UXTH:
943     case AArch64_AM::UXTW:
944     case AArch64_AM::UXTX:
945       return AArch64_AM::getArithShiftValue(Imm) == 0;
946     }
947   }
948 
949   case AArch64::LDRBBroW:
950   case AArch64::LDRBBroX:
951   case AArch64::LDRBroW:
952   case AArch64::LDRBroX:
953   case AArch64::LDRDroW:
954   case AArch64::LDRDroX:
955   case AArch64::LDRHHroW:
956   case AArch64::LDRHHroX:
957   case AArch64::LDRHroW:
958   case AArch64::LDRHroX:
959   case AArch64::LDRQroW:
960   case AArch64::LDRQroX:
961   case AArch64::LDRSBWroW:
962   case AArch64::LDRSBWroX:
963   case AArch64::LDRSBXroW:
964   case AArch64::LDRSBXroX:
965   case AArch64::LDRSHWroW:
966   case AArch64::LDRSHWroX:
967   case AArch64::LDRSHXroW:
968   case AArch64::LDRSHXroX:
969   case AArch64::LDRSWroW:
970   case AArch64::LDRSWroX:
971   case AArch64::LDRSroW:
972   case AArch64::LDRSroX:
973   case AArch64::LDRWroW:
974   case AArch64::LDRWroX:
975   case AArch64::LDRXroW:
976   case AArch64::LDRXroX:
977   case AArch64::PRFMroW:
978   case AArch64::PRFMroX:
979   case AArch64::STRBBroW:
980   case AArch64::STRBBroX:
981   case AArch64::STRBroW:
982   case AArch64::STRBroX:
983   case AArch64::STRDroW:
984   case AArch64::STRDroX:
985   case AArch64::STRHHroW:
986   case AArch64::STRHHroX:
987   case AArch64::STRHroW:
988   case AArch64::STRHroX:
989   case AArch64::STRQroW:
990   case AArch64::STRQroX:
991   case AArch64::STRSroW:
992   case AArch64::STRSroX:
993   case AArch64::STRWroW:
994   case AArch64::STRWroX:
995   case AArch64::STRXroW:
996   case AArch64::STRXroX: {
997     unsigned IsSigned = MI.getOperand(3).getImm();
998     return !IsSigned;
999   }
1000   }
1001 }
1002 
1003 bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1004   unsigned Opc = MI.getOpcode();
1005   switch (Opc) {
1006     default:
1007       return false;
1008     case AArch64::SEH_StackAlloc:
1009     case AArch64::SEH_SaveFPLR:
1010     case AArch64::SEH_SaveFPLR_X:
1011     case AArch64::SEH_SaveReg:
1012     case AArch64::SEH_SaveReg_X:
1013     case AArch64::SEH_SaveRegP:
1014     case AArch64::SEH_SaveRegP_X:
1015     case AArch64::SEH_SaveFReg:
1016     case AArch64::SEH_SaveFReg_X:
1017     case AArch64::SEH_SaveFRegP:
1018     case AArch64::SEH_SaveFRegP_X:
1019     case AArch64::SEH_SetFP:
1020     case AArch64::SEH_AddFP:
1021     case AArch64::SEH_Nop:
1022     case AArch64::SEH_PrologEnd:
1023     case AArch64::SEH_EpilogStart:
1024     case AArch64::SEH_EpilogEnd:
1025       return true;
1026   }
1027 }
1028 
1029 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1030                                              Register &SrcReg, Register &DstReg,
1031                                              unsigned &SubIdx) const {
1032   switch (MI.getOpcode()) {
1033   default:
1034     return false;
1035   case AArch64::SBFMXri: // aka sxtw
1036   case AArch64::UBFMXri: // aka uxtw
1037     // Check for the 32 -> 64 bit extension case, these instructions can do
1038     // much more.
1039     if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
1040       return false;
1041     // This is a signed or unsigned 32 -> 64 bit extension.
1042     SrcReg = MI.getOperand(1).getReg();
1043     DstReg = MI.getOperand(0).getReg();
1044     SubIdx = AArch64::sub_32;
1045     return true;
1046   }
1047 }
1048 
1049 bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1050     const MachineInstr &MIa, const MachineInstr &MIb) const {
1051   const TargetRegisterInfo *TRI = &getRegisterInfo();
1052   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1053   int64_t OffsetA = 0, OffsetB = 0;
1054   unsigned WidthA = 0, WidthB = 0;
1055   bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1056 
1057   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1058   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1059 
1060   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1061       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1062     return false;
1063 
1064   // Retrieve the base, offset from the base and width. Width
1065   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8).  If
1066   // base are identical, and the offset of a lower memory access +
1067   // the width doesn't overlap the offset of a higher memory access,
1068   // then the memory accesses are different.
1069   // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1070   // are assumed to have the same scale (vscale).
1071   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
1072                                    WidthA, TRI) &&
1073       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
1074                                    WidthB, TRI)) {
1075     if (BaseOpA->isIdenticalTo(*BaseOpB) &&
1076         OffsetAIsScalable == OffsetBIsScalable) {
1077       int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1078       int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1079       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1080       if (LowOffset + LowWidth <= HighOffset)
1081         return true;
1082     }
1083   }
1084   return false;
1085 }
1086 
1087 bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1088                                             const MachineBasicBlock *MBB,
1089                                             const MachineFunction &MF) const {
1090   if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1091     return true;
1092   switch (MI.getOpcode()) {
1093   case AArch64::HINT:
1094     // CSDB hints are scheduling barriers.
1095     if (MI.getOperand(0).getImm() == 0x14)
1096       return true;
1097     break;
1098   case AArch64::DSB:
1099   case AArch64::ISB:
1100     // DSB and ISB also are scheduling barriers.
1101     return true;
1102   default:;
1103   }
1104   return isSEHInstruction(MI);
1105 }
1106 
1107 /// analyzeCompare - For a comparison instruction, return the source registers
1108 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1109 /// Return true if the comparison instruction can be analyzed.
1110 bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1111                                       Register &SrcReg2, int &CmpMask,
1112                                       int &CmpValue) const {
1113   // The first operand can be a frame index where we'd normally expect a
1114   // register.
1115   assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1116   if (!MI.getOperand(1).isReg())
1117     return false;
1118 
1119   switch (MI.getOpcode()) {
1120   default:
1121     break;
1122   case AArch64::SUBSWrr:
1123   case AArch64::SUBSWrs:
1124   case AArch64::SUBSWrx:
1125   case AArch64::SUBSXrr:
1126   case AArch64::SUBSXrs:
1127   case AArch64::SUBSXrx:
1128   case AArch64::ADDSWrr:
1129   case AArch64::ADDSWrs:
1130   case AArch64::ADDSWrx:
1131   case AArch64::ADDSXrr:
1132   case AArch64::ADDSXrs:
1133   case AArch64::ADDSXrx:
1134     // Replace SUBSWrr with SUBWrr if NZCV is not used.
1135     SrcReg = MI.getOperand(1).getReg();
1136     SrcReg2 = MI.getOperand(2).getReg();
1137     CmpMask = ~0;
1138     CmpValue = 0;
1139     return true;
1140   case AArch64::SUBSWri:
1141   case AArch64::ADDSWri:
1142   case AArch64::SUBSXri:
1143   case AArch64::ADDSXri:
1144     SrcReg = MI.getOperand(1).getReg();
1145     SrcReg2 = 0;
1146     CmpMask = ~0;
1147     // FIXME: In order to convert CmpValue to 0 or 1
1148     CmpValue = MI.getOperand(2).getImm() != 0;
1149     return true;
1150   case AArch64::ANDSWri:
1151   case AArch64::ANDSXri:
1152     // ANDS does not use the same encoding scheme as the others xxxS
1153     // instructions.
1154     SrcReg = MI.getOperand(1).getReg();
1155     SrcReg2 = 0;
1156     CmpMask = ~0;
1157     // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
1158     // while the type of CmpValue is int. When converting uint64_t to int,
1159     // the high 32 bits of uint64_t will be lost.
1160     // In fact it causes a bug in spec2006-483.xalancbmk
1161     // CmpValue is only used to compare with zero in OptimizeCompareInstr
1162     CmpValue = AArch64_AM::decodeLogicalImmediate(
1163                    MI.getOperand(2).getImm(),
1164                    MI.getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0;
1165     return true;
1166   }
1167 
1168   return false;
1169 }
1170 
1171 static bool UpdateOperandRegClass(MachineInstr &Instr) {
1172   MachineBasicBlock *MBB = Instr.getParent();
1173   assert(MBB && "Can't get MachineBasicBlock here");
1174   MachineFunction *MF = MBB->getParent();
1175   assert(MF && "Can't get MachineFunction here");
1176   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1177   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1178   MachineRegisterInfo *MRI = &MF->getRegInfo();
1179 
1180   for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1181        ++OpIdx) {
1182     MachineOperand &MO = Instr.getOperand(OpIdx);
1183     const TargetRegisterClass *OpRegCstraints =
1184         Instr.getRegClassConstraint(OpIdx, TII, TRI);
1185 
1186     // If there's no constraint, there's nothing to do.
1187     if (!OpRegCstraints)
1188       continue;
1189     // If the operand is a frame index, there's nothing to do here.
1190     // A frame index operand will resolve correctly during PEI.
1191     if (MO.isFI())
1192       continue;
1193 
1194     assert(MO.isReg() &&
1195            "Operand has register constraints without being a register!");
1196 
1197     Register Reg = MO.getReg();
1198     if (Register::isPhysicalRegister(Reg)) {
1199       if (!OpRegCstraints->contains(Reg))
1200         return false;
1201     } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
1202                !MRI->constrainRegClass(Reg, OpRegCstraints))
1203       return false;
1204   }
1205 
1206   return true;
1207 }
1208 
1209 /// Return the opcode that does not set flags when possible - otherwise
1210 /// return the original opcode. The caller is responsible to do the actual
1211 /// substitution and legality checking.
1212 static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1213   // Don't convert all compare instructions, because for some the zero register
1214   // encoding becomes the sp register.
1215   bool MIDefinesZeroReg = false;
1216   if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR))
1217     MIDefinesZeroReg = true;
1218 
1219   switch (MI.getOpcode()) {
1220   default:
1221     return MI.getOpcode();
1222   case AArch64::ADDSWrr:
1223     return AArch64::ADDWrr;
1224   case AArch64::ADDSWri:
1225     return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1226   case AArch64::ADDSWrs:
1227     return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1228   case AArch64::ADDSWrx:
1229     return AArch64::ADDWrx;
1230   case AArch64::ADDSXrr:
1231     return AArch64::ADDXrr;
1232   case AArch64::ADDSXri:
1233     return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1234   case AArch64::ADDSXrs:
1235     return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1236   case AArch64::ADDSXrx:
1237     return AArch64::ADDXrx;
1238   case AArch64::SUBSWrr:
1239     return AArch64::SUBWrr;
1240   case AArch64::SUBSWri:
1241     return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1242   case AArch64::SUBSWrs:
1243     return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1244   case AArch64::SUBSWrx:
1245     return AArch64::SUBWrx;
1246   case AArch64::SUBSXrr:
1247     return AArch64::SUBXrr;
1248   case AArch64::SUBSXri:
1249     return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1250   case AArch64::SUBSXrs:
1251     return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1252   case AArch64::SUBSXrx:
1253     return AArch64::SUBXrx;
1254   }
1255 }
1256 
1257 enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1258 
1259 /// True when condition flags are accessed (either by writing or reading)
1260 /// on the instruction trace starting at From and ending at To.
1261 ///
1262 /// Note: If From and To are from different blocks it's assumed CC are accessed
1263 ///       on the path.
1264 static bool areCFlagsAccessedBetweenInstrs(
1265     MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1266     const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1267   // Early exit if To is at the beginning of the BB.
1268   if (To == To->getParent()->begin())
1269     return true;
1270 
1271   // Check whether the instructions are in the same basic block
1272   // If not, assume the condition flags might get modified somewhere.
1273   if (To->getParent() != From->getParent())
1274     return true;
1275 
1276   // From must be above To.
1277   assert(std::find_if(++To.getReverse(), To->getParent()->rend(),
1278                       [From](MachineInstr &MI) {
1279                         return MI.getIterator() == From;
1280                       }) != To->getParent()->rend());
1281 
1282   // We iterate backward starting at \p To until we hit \p From.
1283   for (const MachineInstr &Instr :
1284        instructionsWithoutDebug(++To.getReverse(), From.getReverse())) {
1285     if (((AccessToCheck & AK_Write) &&
1286          Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
1287         ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
1288       return true;
1289   }
1290   return false;
1291 }
1292 
1293 /// Try to optimize a compare instruction. A compare instruction is an
1294 /// instruction which produces AArch64::NZCV. It can be truly compare
1295 /// instruction
1296 /// when there are no uses of its destination register.
1297 ///
1298 /// The following steps are tried in order:
1299 /// 1. Convert CmpInstr into an unconditional version.
1300 /// 2. Remove CmpInstr if above there is an instruction producing a needed
1301 ///    condition code or an instruction which can be converted into such an
1302 ///    instruction.
1303 ///    Only comparison with zero is supported.
1304 bool AArch64InstrInfo::optimizeCompareInstr(
1305     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
1306     int CmpValue, const MachineRegisterInfo *MRI) const {
1307   assert(CmpInstr.getParent());
1308   assert(MRI);
1309 
1310   // Replace SUBSWrr with SUBWrr if NZCV is not used.
1311   int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true);
1312   if (DeadNZCVIdx != -1) {
1313     if (CmpInstr.definesRegister(AArch64::WZR) ||
1314         CmpInstr.definesRegister(AArch64::XZR)) {
1315       CmpInstr.eraseFromParent();
1316       return true;
1317     }
1318     unsigned Opc = CmpInstr.getOpcode();
1319     unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
1320     if (NewOpc == Opc)
1321       return false;
1322     const MCInstrDesc &MCID = get(NewOpc);
1323     CmpInstr.setDesc(MCID);
1324     CmpInstr.RemoveOperand(DeadNZCVIdx);
1325     bool succeeded = UpdateOperandRegClass(CmpInstr);
1326     (void)succeeded;
1327     assert(succeeded && "Some operands reg class are incompatible!");
1328     return true;
1329   }
1330 
1331   // Continue only if we have a "ri" where immediate is zero.
1332   // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
1333   // function.
1334   assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
1335   if (CmpValue != 0 || SrcReg2 != 0)
1336     return false;
1337 
1338   // CmpInstr is a Compare instruction if destination register is not used.
1339   if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
1340     return false;
1341 
1342   return substituteCmpToZero(CmpInstr, SrcReg, MRI);
1343 }
1344 
1345 /// Get opcode of S version of Instr.
1346 /// If Instr is S version its opcode is returned.
1347 /// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
1348 /// or we are not interested in it.
1349 static unsigned sForm(MachineInstr &Instr) {
1350   switch (Instr.getOpcode()) {
1351   default:
1352     return AArch64::INSTRUCTION_LIST_END;
1353 
1354   case AArch64::ADDSWrr:
1355   case AArch64::ADDSWri:
1356   case AArch64::ADDSXrr:
1357   case AArch64::ADDSXri:
1358   case AArch64::SUBSWrr:
1359   case AArch64::SUBSWri:
1360   case AArch64::SUBSXrr:
1361   case AArch64::SUBSXri:
1362     return Instr.getOpcode();
1363 
1364   case AArch64::ADDWrr:
1365     return AArch64::ADDSWrr;
1366   case AArch64::ADDWri:
1367     return AArch64::ADDSWri;
1368   case AArch64::ADDXrr:
1369     return AArch64::ADDSXrr;
1370   case AArch64::ADDXri:
1371     return AArch64::ADDSXri;
1372   case AArch64::ADCWr:
1373     return AArch64::ADCSWr;
1374   case AArch64::ADCXr:
1375     return AArch64::ADCSXr;
1376   case AArch64::SUBWrr:
1377     return AArch64::SUBSWrr;
1378   case AArch64::SUBWri:
1379     return AArch64::SUBSWri;
1380   case AArch64::SUBXrr:
1381     return AArch64::SUBSXrr;
1382   case AArch64::SUBXri:
1383     return AArch64::SUBSXri;
1384   case AArch64::SBCWr:
1385     return AArch64::SBCSWr;
1386   case AArch64::SBCXr:
1387     return AArch64::SBCSXr;
1388   case AArch64::ANDWri:
1389     return AArch64::ANDSWri;
1390   case AArch64::ANDXri:
1391     return AArch64::ANDSXri;
1392   }
1393 }
1394 
1395 /// Check if AArch64::NZCV should be alive in successors of MBB.
1396 static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) {
1397   for (auto *BB : MBB->successors())
1398     if (BB->isLiveIn(AArch64::NZCV))
1399       return true;
1400   return false;
1401 }
1402 
1403 namespace {
1404 
1405 struct UsedNZCV {
1406   bool N = false;
1407   bool Z = false;
1408   bool C = false;
1409   bool V = false;
1410 
1411   UsedNZCV() = default;
1412 
1413   UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
1414     this->N |= UsedFlags.N;
1415     this->Z |= UsedFlags.Z;
1416     this->C |= UsedFlags.C;
1417     this->V |= UsedFlags.V;
1418     return *this;
1419   }
1420 };
1421 
1422 } // end anonymous namespace
1423 
1424 /// Find a condition code used by the instruction.
1425 /// Returns AArch64CC::Invalid if either the instruction does not use condition
1426 /// codes or we don't optimize CmpInstr in the presence of such instructions.
1427 static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
1428   switch (Instr.getOpcode()) {
1429   default:
1430     return AArch64CC::Invalid;
1431 
1432   case AArch64::Bcc: {
1433     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1434     assert(Idx >= 2);
1435     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 2).getImm());
1436   }
1437 
1438   case AArch64::CSINVWr:
1439   case AArch64::CSINVXr:
1440   case AArch64::CSINCWr:
1441   case AArch64::CSINCXr:
1442   case AArch64::CSELWr:
1443   case AArch64::CSELXr:
1444   case AArch64::CSNEGWr:
1445   case AArch64::CSNEGXr:
1446   case AArch64::FCSELSrrr:
1447   case AArch64::FCSELDrrr: {
1448     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1449     assert(Idx >= 1);
1450     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 1).getImm());
1451   }
1452   }
1453 }
1454 
1455 static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
1456   assert(CC != AArch64CC::Invalid);
1457   UsedNZCV UsedFlags;
1458   switch (CC) {
1459   default:
1460     break;
1461 
1462   case AArch64CC::EQ: // Z set
1463   case AArch64CC::NE: // Z clear
1464     UsedFlags.Z = true;
1465     break;
1466 
1467   case AArch64CC::HI: // Z clear and C set
1468   case AArch64CC::LS: // Z set   or  C clear
1469     UsedFlags.Z = true;
1470     LLVM_FALLTHROUGH;
1471   case AArch64CC::HS: // C set
1472   case AArch64CC::LO: // C clear
1473     UsedFlags.C = true;
1474     break;
1475 
1476   case AArch64CC::MI: // N set
1477   case AArch64CC::PL: // N clear
1478     UsedFlags.N = true;
1479     break;
1480 
1481   case AArch64CC::VS: // V set
1482   case AArch64CC::VC: // V clear
1483     UsedFlags.V = true;
1484     break;
1485 
1486   case AArch64CC::GT: // Z clear, N and V the same
1487   case AArch64CC::LE: // Z set,   N and V differ
1488     UsedFlags.Z = true;
1489     LLVM_FALLTHROUGH;
1490   case AArch64CC::GE: // N and V the same
1491   case AArch64CC::LT: // N and V differ
1492     UsedFlags.N = true;
1493     UsedFlags.V = true;
1494     break;
1495   }
1496   return UsedFlags;
1497 }
1498 
1499 static bool isADDSRegImm(unsigned Opcode) {
1500   return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1501 }
1502 
1503 static bool isSUBSRegImm(unsigned Opcode) {
1504   return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1505 }
1506 
1507 /// Check if CmpInstr can be substituted by MI.
1508 ///
1509 /// CmpInstr can be substituted:
1510 /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1511 /// - and, MI and CmpInstr are from the same MachineBB
1512 /// - and, condition flags are not alive in successors of the CmpInstr parent
1513 /// - and, if MI opcode is the S form there must be no defs of flags between
1514 ///        MI and CmpInstr
1515 ///        or if MI opcode is not the S form there must be neither defs of flags
1516 ///        nor uses of flags between MI and CmpInstr.
1517 /// - and  C/V flags are not used after CmpInstr
1518 static bool canInstrSubstituteCmpInstr(MachineInstr *MI, MachineInstr *CmpInstr,
1519                                        const TargetRegisterInfo *TRI) {
1520   assert(MI);
1521   assert(sForm(*MI) != AArch64::INSTRUCTION_LIST_END);
1522   assert(CmpInstr);
1523 
1524   const unsigned CmpOpcode = CmpInstr->getOpcode();
1525   if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
1526     return false;
1527 
1528   if (MI->getParent() != CmpInstr->getParent())
1529     return false;
1530 
1531   if (areCFlagsAliveInSuccessors(CmpInstr->getParent()))
1532     return false;
1533 
1534   AccessKind AccessToCheck = AK_Write;
1535   if (sForm(*MI) != MI->getOpcode())
1536     AccessToCheck = AK_All;
1537   if (areCFlagsAccessedBetweenInstrs(MI, CmpInstr, TRI, AccessToCheck))
1538     return false;
1539 
1540   UsedNZCV NZCVUsedAfterCmp;
1541   for (const MachineInstr &Instr :
1542        instructionsWithoutDebug(std::next(CmpInstr->getIterator()),
1543                                 CmpInstr->getParent()->instr_end())) {
1544     if (Instr.readsRegister(AArch64::NZCV, TRI)) {
1545       AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
1546       if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
1547         return false;
1548       NZCVUsedAfterCmp |= getUsedNZCV(CC);
1549     }
1550 
1551     if (Instr.modifiesRegister(AArch64::NZCV, TRI))
1552       break;
1553   }
1554 
1555   return !NZCVUsedAfterCmp.C && !NZCVUsedAfterCmp.V;
1556 }
1557 
1558 /// Substitute an instruction comparing to zero with another instruction
1559 /// which produces needed condition flags.
1560 ///
1561 /// Return true on success.
1562 bool AArch64InstrInfo::substituteCmpToZero(
1563     MachineInstr &CmpInstr, unsigned SrcReg,
1564     const MachineRegisterInfo *MRI) const {
1565   assert(MRI);
1566   // Get the unique definition of SrcReg.
1567   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1568   if (!MI)
1569     return false;
1570 
1571   const TargetRegisterInfo *TRI = &getRegisterInfo();
1572 
1573   unsigned NewOpc = sForm(*MI);
1574   if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1575     return false;
1576 
1577   if (!canInstrSubstituteCmpInstr(MI, &CmpInstr, TRI))
1578     return false;
1579 
1580   // Update the instruction to set NZCV.
1581   MI->setDesc(get(NewOpc));
1582   CmpInstr.eraseFromParent();
1583   bool succeeded = UpdateOperandRegClass(*MI);
1584   (void)succeeded;
1585   assert(succeeded && "Some operands reg class are incompatible!");
1586   MI->addRegisterDefined(AArch64::NZCV, TRI);
1587   return true;
1588 }
1589 
1590 bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1591   if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1592       MI.getOpcode() != AArch64::CATCHRET)
1593     return false;
1594 
1595   MachineBasicBlock &MBB = *MI.getParent();
1596   auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1597   auto TRI = Subtarget.getRegisterInfo();
1598   DebugLoc DL = MI.getDebugLoc();
1599 
1600   if (MI.getOpcode() == AArch64::CATCHRET) {
1601     // Skip to the first instruction before the epilog.
1602     const TargetInstrInfo *TII =
1603       MBB.getParent()->getSubtarget().getInstrInfo();
1604     MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
1605     auto MBBI = MachineBasicBlock::iterator(MI);
1606     MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
1607     while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
1608            FirstEpilogSEH != MBB.begin())
1609       FirstEpilogSEH = std::prev(FirstEpilogSEH);
1610     if (FirstEpilogSEH != MBB.begin())
1611       FirstEpilogSEH = std::next(FirstEpilogSEH);
1612     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
1613         .addReg(AArch64::X0, RegState::Define)
1614         .addMBB(TargetMBB);
1615     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
1616         .addReg(AArch64::X0, RegState::Define)
1617         .addReg(AArch64::X0)
1618         .addMBB(TargetMBB)
1619         .addImm(0);
1620     return true;
1621   }
1622 
1623   Register Reg = MI.getOperand(0).getReg();
1624   const GlobalValue *GV =
1625       cast<GlobalValue>((*MI.memoperands_begin())->getValue());
1626   const TargetMachine &TM = MBB.getParent()->getTarget();
1627   unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
1628   const unsigned char MO_NC = AArch64II::MO_NC;
1629 
1630   if ((OpFlags & AArch64II::MO_GOT) != 0) {
1631     BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
1632         .addGlobalAddress(GV, 0, OpFlags);
1633     if (Subtarget.isTargetILP32()) {
1634       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1635       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1636           .addDef(Reg32, RegState::Dead)
1637           .addUse(Reg, RegState::Kill)
1638           .addImm(0)
1639           .addMemOperand(*MI.memoperands_begin())
1640           .addDef(Reg, RegState::Implicit);
1641     } else {
1642       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1643           .addReg(Reg, RegState::Kill)
1644           .addImm(0)
1645           .addMemOperand(*MI.memoperands_begin());
1646     }
1647   } else if (TM.getCodeModel() == CodeModel::Large) {
1648     assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?");
1649     BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
1650         .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
1651         .addImm(0);
1652     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1653         .addReg(Reg, RegState::Kill)
1654         .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
1655         .addImm(16);
1656     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1657         .addReg(Reg, RegState::Kill)
1658         .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
1659         .addImm(32);
1660     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1661         .addReg(Reg, RegState::Kill)
1662         .addGlobalAddress(GV, 0, AArch64II::MO_G3)
1663         .addImm(48);
1664     BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1665         .addReg(Reg, RegState::Kill)
1666         .addImm(0)
1667         .addMemOperand(*MI.memoperands_begin());
1668   } else if (TM.getCodeModel() == CodeModel::Tiny) {
1669     BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg)
1670         .addGlobalAddress(GV, 0, OpFlags);
1671   } else {
1672     BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
1673         .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
1674     unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
1675     if (Subtarget.isTargetILP32()) {
1676       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1677       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1678           .addDef(Reg32, RegState::Dead)
1679           .addUse(Reg, RegState::Kill)
1680           .addGlobalAddress(GV, 0, LoFlags)
1681           .addMemOperand(*MI.memoperands_begin())
1682           .addDef(Reg, RegState::Implicit);
1683     } else {
1684       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1685           .addReg(Reg, RegState::Kill)
1686           .addGlobalAddress(GV, 0, LoFlags)
1687           .addMemOperand(*MI.memoperands_begin());
1688     }
1689   }
1690 
1691   MBB.erase(MI);
1692 
1693   return true;
1694 }
1695 
1696 // Return true if this instruction simply sets its single destination register
1697 // to zero. This is equivalent to a register rename of the zero-register.
1698 bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
1699   switch (MI.getOpcode()) {
1700   default:
1701     break;
1702   case AArch64::MOVZWi:
1703   case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1704     if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
1705       assert(MI.getDesc().getNumOperands() == 3 &&
1706              MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1707       return true;
1708     }
1709     break;
1710   case AArch64::ANDWri: // and Rd, Rzr, #imm
1711     return MI.getOperand(1).getReg() == AArch64::WZR;
1712   case AArch64::ANDXri:
1713     return MI.getOperand(1).getReg() == AArch64::XZR;
1714   case TargetOpcode::COPY:
1715     return MI.getOperand(1).getReg() == AArch64::WZR;
1716   }
1717   return false;
1718 }
1719 
1720 // Return true if this instruction simply renames a general register without
1721 // modifying bits.
1722 bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
1723   switch (MI.getOpcode()) {
1724   default:
1725     break;
1726   case TargetOpcode::COPY: {
1727     // GPR32 copies will by lowered to ORRXrs
1728     Register DstReg = MI.getOperand(0).getReg();
1729     return (AArch64::GPR32RegClass.contains(DstReg) ||
1730             AArch64::GPR64RegClass.contains(DstReg));
1731   }
1732   case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1733     if (MI.getOperand(1).getReg() == AArch64::XZR) {
1734       assert(MI.getDesc().getNumOperands() == 4 &&
1735              MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1736       return true;
1737     }
1738     break;
1739   case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1740     if (MI.getOperand(2).getImm() == 0) {
1741       assert(MI.getDesc().getNumOperands() == 4 &&
1742              MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1743       return true;
1744     }
1745     break;
1746   }
1747   return false;
1748 }
1749 
1750 // Return true if this instruction simply renames a general register without
1751 // modifying bits.
1752 bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
1753   switch (MI.getOpcode()) {
1754   default:
1755     break;
1756   case TargetOpcode::COPY: {
1757     // FPR64 copies will by lowered to ORR.16b
1758     Register DstReg = MI.getOperand(0).getReg();
1759     return (AArch64::FPR64RegClass.contains(DstReg) ||
1760             AArch64::FPR128RegClass.contains(DstReg));
1761   }
1762   case AArch64::ORRv16i8:
1763     if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
1764       assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
1765              "invalid ORRv16i8 operands");
1766       return true;
1767     }
1768     break;
1769   }
1770   return false;
1771 }
1772 
1773 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1774                                                int &FrameIndex) const {
1775   switch (MI.getOpcode()) {
1776   default:
1777     break;
1778   case AArch64::LDRWui:
1779   case AArch64::LDRXui:
1780   case AArch64::LDRBui:
1781   case AArch64::LDRHui:
1782   case AArch64::LDRSui:
1783   case AArch64::LDRDui:
1784   case AArch64::LDRQui:
1785     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1786         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1787       FrameIndex = MI.getOperand(1).getIndex();
1788       return MI.getOperand(0).getReg();
1789     }
1790     break;
1791   }
1792 
1793   return 0;
1794 }
1795 
1796 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1797                                               int &FrameIndex) const {
1798   switch (MI.getOpcode()) {
1799   default:
1800     break;
1801   case AArch64::STRWui:
1802   case AArch64::STRXui:
1803   case AArch64::STRBui:
1804   case AArch64::STRHui:
1805   case AArch64::STRSui:
1806   case AArch64::STRDui:
1807   case AArch64::STRQui:
1808   case AArch64::LDR_PXI:
1809   case AArch64::STR_PXI:
1810     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1811         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1812       FrameIndex = MI.getOperand(1).getIndex();
1813       return MI.getOperand(0).getReg();
1814     }
1815     break;
1816   }
1817   return 0;
1818 }
1819 
1820 /// Check all MachineMemOperands for a hint to suppress pairing.
1821 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
1822   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1823     return MMO->getFlags() & MOSuppressPair;
1824   });
1825 }
1826 
1827 /// Set a flag on the first MachineMemOperand to suppress pairing.
1828 void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
1829   if (MI.memoperands_empty())
1830     return;
1831   (*MI.memoperands_begin())->setFlags(MOSuppressPair);
1832 }
1833 
1834 /// Check all MachineMemOperands for a hint that the load/store is strided.
1835 bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
1836   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1837     return MMO->getFlags() & MOStridedAccess;
1838   });
1839 }
1840 
1841 bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) {
1842   switch (Opc) {
1843   default:
1844     return false;
1845   case AArch64::STURSi:
1846   case AArch64::STURDi:
1847   case AArch64::STURQi:
1848   case AArch64::STURBBi:
1849   case AArch64::STURHHi:
1850   case AArch64::STURWi:
1851   case AArch64::STURXi:
1852   case AArch64::LDURSi:
1853   case AArch64::LDURDi:
1854   case AArch64::LDURQi:
1855   case AArch64::LDURWi:
1856   case AArch64::LDURXi:
1857   case AArch64::LDURSWi:
1858   case AArch64::LDURHHi:
1859   case AArch64::LDURBBi:
1860   case AArch64::LDURSBWi:
1861   case AArch64::LDURSHWi:
1862     return true;
1863   }
1864 }
1865 
1866 Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
1867   switch (Opc) {
1868   default: return {};
1869   case AArch64::PRFMui: return AArch64::PRFUMi;
1870   case AArch64::LDRXui: return AArch64::LDURXi;
1871   case AArch64::LDRWui: return AArch64::LDURWi;
1872   case AArch64::LDRBui: return AArch64::LDURBi;
1873   case AArch64::LDRHui: return AArch64::LDURHi;
1874   case AArch64::LDRSui: return AArch64::LDURSi;
1875   case AArch64::LDRDui: return AArch64::LDURDi;
1876   case AArch64::LDRQui: return AArch64::LDURQi;
1877   case AArch64::LDRBBui: return AArch64::LDURBBi;
1878   case AArch64::LDRHHui: return AArch64::LDURHHi;
1879   case AArch64::LDRSBXui: return AArch64::LDURSBXi;
1880   case AArch64::LDRSBWui: return AArch64::LDURSBWi;
1881   case AArch64::LDRSHXui: return AArch64::LDURSHXi;
1882   case AArch64::LDRSHWui: return AArch64::LDURSHWi;
1883   case AArch64::LDRSWui: return AArch64::LDURSWi;
1884   case AArch64::STRXui: return AArch64::STURXi;
1885   case AArch64::STRWui: return AArch64::STURWi;
1886   case AArch64::STRBui: return AArch64::STURBi;
1887   case AArch64::STRHui: return AArch64::STURHi;
1888   case AArch64::STRSui: return AArch64::STURSi;
1889   case AArch64::STRDui: return AArch64::STURDi;
1890   case AArch64::STRQui: return AArch64::STURQi;
1891   case AArch64::STRBBui: return AArch64::STURBBi;
1892   case AArch64::STRHHui: return AArch64::STURHHi;
1893   }
1894 }
1895 
1896 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
1897   switch (Opc) {
1898   default:
1899     return 2;
1900   case AArch64::LDPXi:
1901   case AArch64::LDPDi:
1902   case AArch64::STPXi:
1903   case AArch64::STPDi:
1904   case AArch64::LDNPXi:
1905   case AArch64::LDNPDi:
1906   case AArch64::STNPXi:
1907   case AArch64::STNPDi:
1908   case AArch64::LDPQi:
1909   case AArch64::STPQi:
1910   case AArch64::LDNPQi:
1911   case AArch64::STNPQi:
1912   case AArch64::LDPWi:
1913   case AArch64::LDPSi:
1914   case AArch64::STPWi:
1915   case AArch64::STPSi:
1916   case AArch64::LDNPWi:
1917   case AArch64::LDNPSi:
1918   case AArch64::STNPWi:
1919   case AArch64::STNPSi:
1920   case AArch64::LDG:
1921   case AArch64::STGPi:
1922   case AArch64::LD1B_IMM:
1923   case AArch64::LD1H_IMM:
1924   case AArch64::LD1W_IMM:
1925   case AArch64::LD1D_IMM:
1926   case AArch64::ST1B_IMM:
1927   case AArch64::ST1H_IMM:
1928   case AArch64::ST1W_IMM:
1929   case AArch64::ST1D_IMM:
1930   case AArch64::LD1B_H_IMM:
1931   case AArch64::LD1SB_H_IMM:
1932   case AArch64::LD1H_S_IMM:
1933   case AArch64::LD1SH_S_IMM:
1934   case AArch64::LD1W_D_IMM:
1935   case AArch64::LD1SW_D_IMM:
1936   case AArch64::ST1B_H_IMM:
1937   case AArch64::ST1H_S_IMM:
1938   case AArch64::ST1W_D_IMM:
1939   case AArch64::LD1B_S_IMM:
1940   case AArch64::LD1SB_S_IMM:
1941   case AArch64::LD1H_D_IMM:
1942   case AArch64::LD1SH_D_IMM:
1943   case AArch64::ST1B_S_IMM:
1944   case AArch64::ST1H_D_IMM:
1945   case AArch64::LD1B_D_IMM:
1946   case AArch64::LD1SB_D_IMM:
1947   case AArch64::ST1B_D_IMM:
1948     return 3;
1949   case AArch64::ADDG:
1950   case AArch64::STGOffset:
1951   case AArch64::LDR_PXI:
1952   case AArch64::STR_PXI:
1953     return 2;
1954   }
1955 }
1956 
1957 bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
1958   switch (MI.getOpcode()) {
1959   default:
1960     return false;
1961   // Scaled instructions.
1962   case AArch64::STRSui:
1963   case AArch64::STRDui:
1964   case AArch64::STRQui:
1965   case AArch64::STRXui:
1966   case AArch64::STRWui:
1967   case AArch64::LDRSui:
1968   case AArch64::LDRDui:
1969   case AArch64::LDRQui:
1970   case AArch64::LDRXui:
1971   case AArch64::LDRWui:
1972   case AArch64::LDRSWui:
1973   // Unscaled instructions.
1974   case AArch64::STURSi:
1975   case AArch64::STURDi:
1976   case AArch64::STURQi:
1977   case AArch64::STURWi:
1978   case AArch64::STURXi:
1979   case AArch64::LDURSi:
1980   case AArch64::LDURDi:
1981   case AArch64::LDURQi:
1982   case AArch64::LDURWi:
1983   case AArch64::LDURXi:
1984   case AArch64::LDURSWi:
1985     return true;
1986   }
1987 }
1988 
1989 unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc,
1990                                                    bool &Is64Bit) {
1991   switch (Opc) {
1992   default:
1993     llvm_unreachable("Opcode has no flag setting equivalent!");
1994   // 32-bit cases:
1995   case AArch64::ADDWri:
1996     Is64Bit = false;
1997     return AArch64::ADDSWri;
1998   case AArch64::ADDWrr:
1999     Is64Bit = false;
2000     return AArch64::ADDSWrr;
2001   case AArch64::ADDWrs:
2002     Is64Bit = false;
2003     return AArch64::ADDSWrs;
2004   case AArch64::ADDWrx:
2005     Is64Bit = false;
2006     return AArch64::ADDSWrx;
2007   case AArch64::ANDWri:
2008     Is64Bit = false;
2009     return AArch64::ANDSWri;
2010   case AArch64::ANDWrr:
2011     Is64Bit = false;
2012     return AArch64::ANDSWrr;
2013   case AArch64::ANDWrs:
2014     Is64Bit = false;
2015     return AArch64::ANDSWrs;
2016   case AArch64::BICWrr:
2017     Is64Bit = false;
2018     return AArch64::BICSWrr;
2019   case AArch64::BICWrs:
2020     Is64Bit = false;
2021     return AArch64::BICSWrs;
2022   case AArch64::SUBWri:
2023     Is64Bit = false;
2024     return AArch64::SUBSWri;
2025   case AArch64::SUBWrr:
2026     Is64Bit = false;
2027     return AArch64::SUBSWrr;
2028   case AArch64::SUBWrs:
2029     Is64Bit = false;
2030     return AArch64::SUBSWrs;
2031   case AArch64::SUBWrx:
2032     Is64Bit = false;
2033     return AArch64::SUBSWrx;
2034   // 64-bit cases:
2035   case AArch64::ADDXri:
2036     Is64Bit = true;
2037     return AArch64::ADDSXri;
2038   case AArch64::ADDXrr:
2039     Is64Bit = true;
2040     return AArch64::ADDSXrr;
2041   case AArch64::ADDXrs:
2042     Is64Bit = true;
2043     return AArch64::ADDSXrs;
2044   case AArch64::ADDXrx:
2045     Is64Bit = true;
2046     return AArch64::ADDSXrx;
2047   case AArch64::ANDXri:
2048     Is64Bit = true;
2049     return AArch64::ANDSXri;
2050   case AArch64::ANDXrr:
2051     Is64Bit = true;
2052     return AArch64::ANDSXrr;
2053   case AArch64::ANDXrs:
2054     Is64Bit = true;
2055     return AArch64::ANDSXrs;
2056   case AArch64::BICXrr:
2057     Is64Bit = true;
2058     return AArch64::BICSXrr;
2059   case AArch64::BICXrs:
2060     Is64Bit = true;
2061     return AArch64::BICSXrs;
2062   case AArch64::SUBXri:
2063     Is64Bit = true;
2064     return AArch64::SUBSXri;
2065   case AArch64::SUBXrr:
2066     Is64Bit = true;
2067     return AArch64::SUBSXrr;
2068   case AArch64::SUBXrs:
2069     Is64Bit = true;
2070     return AArch64::SUBSXrs;
2071   case AArch64::SUBXrx:
2072     Is64Bit = true;
2073     return AArch64::SUBSXrx;
2074   }
2075 }
2076 
2077 // Is this a candidate for ld/st merging or pairing?  For example, we don't
2078 // touch volatiles or load/stores that have a hint to avoid pair formation.
2079 bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
2080   // If this is a volatile load/store, don't mess with it.
2081   if (MI.hasOrderedMemoryRef())
2082     return false;
2083 
2084   // Make sure this is a reg/fi+imm (as opposed to an address reloc).
2085   assert((MI.getOperand(1).isReg() || MI.getOperand(1).isFI()) &&
2086          "Expected a reg or frame index operand.");
2087   if (!MI.getOperand(2).isImm())
2088     return false;
2089 
2090   // Can't merge/pair if the instruction modifies the base register.
2091   // e.g., ldr x0, [x0]
2092   // This case will never occur with an FI base.
2093   if (MI.getOperand(1).isReg()) {
2094     Register BaseReg = MI.getOperand(1).getReg();
2095     const TargetRegisterInfo *TRI = &getRegisterInfo();
2096     if (MI.modifiesRegister(BaseReg, TRI))
2097       return false;
2098   }
2099 
2100   // Check if this load/store has a hint to avoid pair formation.
2101   // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
2102   if (isLdStPairSuppressed(MI))
2103     return false;
2104 
2105   // Do not pair any callee-save store/reload instructions in the
2106   // prologue/epilogue if the CFI information encoded the operations as separate
2107   // instructions, as that will cause the size of the actual prologue to mismatch
2108   // with the prologue size recorded in the Windows CFI.
2109   const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo();
2110   bool NeedsWinCFI = MAI->usesWindowsCFI() &&
2111                      MI.getMF()->getFunction().needsUnwindTableEntry();
2112   if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
2113                       MI.getFlag(MachineInstr::FrameDestroy)))
2114     return false;
2115 
2116   // On some CPUs quad load/store pairs are slower than two single load/stores.
2117   if (Subtarget.isPaired128Slow()) {
2118     switch (MI.getOpcode()) {
2119     default:
2120       break;
2121     case AArch64::LDURQi:
2122     case AArch64::STURQi:
2123     case AArch64::LDRQui:
2124     case AArch64::STRQui:
2125       return false;
2126     }
2127   }
2128 
2129   return true;
2130 }
2131 
2132 bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
2133     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2134     int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2135     const TargetRegisterInfo *TRI) const {
2136   if (!LdSt.mayLoadOrStore())
2137     return false;
2138 
2139   const MachineOperand *BaseOp;
2140   if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
2141                                     Width, TRI))
2142     return false;
2143   BaseOps.push_back(BaseOp);
2144   return true;
2145 }
2146 
2147 Optional<ExtAddrMode>
2148 AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
2149                                           const TargetRegisterInfo *TRI) const {
2150   const MachineOperand *Base; // Filled with the base operand of MI.
2151   int64_t Offset;             // Filled with the offset of MI.
2152   bool OffsetIsScalable;
2153   if (!getMemOperandWithOffset(MemI, Base, Offset, OffsetIsScalable, TRI))
2154     return None;
2155 
2156   if (!Base->isReg())
2157     return None;
2158   ExtAddrMode AM;
2159   AM.BaseReg = Base->getReg();
2160   AM.Displacement = Offset;
2161   AM.ScaledReg = 0;
2162   return AM;
2163 }
2164 
2165 bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
2166     const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
2167     bool &OffsetIsScalable, unsigned &Width,
2168     const TargetRegisterInfo *TRI) const {
2169   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2170   // Handle only loads/stores with base register followed by immediate offset.
2171   if (LdSt.getNumExplicitOperands() == 3) {
2172     // Non-paired instruction (e.g., ldr x1, [x0, #8]).
2173     if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2174         !LdSt.getOperand(2).isImm())
2175       return false;
2176   } else if (LdSt.getNumExplicitOperands() == 4) {
2177     // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
2178     if (!LdSt.getOperand(1).isReg() ||
2179         (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
2180         !LdSt.getOperand(3).isImm())
2181       return false;
2182   } else
2183     return false;
2184 
2185   // Get the scaling factor for the instruction and set the width for the
2186   // instruction.
2187   TypeSize Scale(0U, false);
2188   int64_t Dummy1, Dummy2;
2189 
2190   // If this returns false, then it's an instruction we don't want to handle.
2191   if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
2192     return false;
2193 
2194   // Compute the offset. Offset is calculated as the immediate operand
2195   // multiplied by the scaling factor. Unscaled instructions have scaling factor
2196   // set to 1.
2197   if (LdSt.getNumExplicitOperands() == 3) {
2198     BaseOp = &LdSt.getOperand(1);
2199     Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinSize();
2200   } else {
2201     assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
2202     BaseOp = &LdSt.getOperand(2);
2203     Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinSize();
2204   }
2205   OffsetIsScalable = Scale.isScalable();
2206 
2207   if (!BaseOp->isReg() && !BaseOp->isFI())
2208     return false;
2209 
2210   return true;
2211 }
2212 
2213 MachineOperand &
2214 AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
2215   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2216   MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
2217   assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
2218   return OfsOp;
2219 }
2220 
2221 bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
2222                                     unsigned &Width, int64_t &MinOffset,
2223                                     int64_t &MaxOffset) {
2224   const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
2225   switch (Opcode) {
2226   // Not a memory operation or something we want to handle.
2227   default:
2228     Scale = TypeSize::Fixed(0);
2229     Width = 0;
2230     MinOffset = MaxOffset = 0;
2231     return false;
2232   case AArch64::STRWpost:
2233   case AArch64::LDRWpost:
2234     Width = 32;
2235     Scale = TypeSize::Fixed(4);
2236     MinOffset = -256;
2237     MaxOffset = 255;
2238     break;
2239   case AArch64::LDURQi:
2240   case AArch64::STURQi:
2241     Width = 16;
2242     Scale = TypeSize::Fixed(1);
2243     MinOffset = -256;
2244     MaxOffset = 255;
2245     break;
2246   case AArch64::PRFUMi:
2247   case AArch64::LDURXi:
2248   case AArch64::LDURDi:
2249   case AArch64::STURXi:
2250   case AArch64::STURDi:
2251     Width = 8;
2252     Scale = TypeSize::Fixed(1);
2253     MinOffset = -256;
2254     MaxOffset = 255;
2255     break;
2256   case AArch64::LDURWi:
2257   case AArch64::LDURSi:
2258   case AArch64::LDURSWi:
2259   case AArch64::STURWi:
2260   case AArch64::STURSi:
2261     Width = 4;
2262     Scale = TypeSize::Fixed(1);
2263     MinOffset = -256;
2264     MaxOffset = 255;
2265     break;
2266   case AArch64::LDURHi:
2267   case AArch64::LDURHHi:
2268   case AArch64::LDURSHXi:
2269   case AArch64::LDURSHWi:
2270   case AArch64::STURHi:
2271   case AArch64::STURHHi:
2272     Width = 2;
2273     Scale = TypeSize::Fixed(1);
2274     MinOffset = -256;
2275     MaxOffset = 255;
2276     break;
2277   case AArch64::LDURBi:
2278   case AArch64::LDURBBi:
2279   case AArch64::LDURSBXi:
2280   case AArch64::LDURSBWi:
2281   case AArch64::STURBi:
2282   case AArch64::STURBBi:
2283     Width = 1;
2284     Scale = TypeSize::Fixed(1);
2285     MinOffset = -256;
2286     MaxOffset = 255;
2287     break;
2288   case AArch64::LDPQi:
2289   case AArch64::LDNPQi:
2290   case AArch64::STPQi:
2291   case AArch64::STNPQi:
2292     Scale = TypeSize::Fixed(16);
2293     Width = 32;
2294     MinOffset = -64;
2295     MaxOffset = 63;
2296     break;
2297   case AArch64::LDRQui:
2298   case AArch64::STRQui:
2299     Scale = TypeSize::Fixed(16);
2300     Width = 16;
2301     MinOffset = 0;
2302     MaxOffset = 4095;
2303     break;
2304   case AArch64::LDPXi:
2305   case AArch64::LDPDi:
2306   case AArch64::LDNPXi:
2307   case AArch64::LDNPDi:
2308   case AArch64::STPXi:
2309   case AArch64::STPDi:
2310   case AArch64::STNPXi:
2311   case AArch64::STNPDi:
2312     Scale = TypeSize::Fixed(8);
2313     Width = 16;
2314     MinOffset = -64;
2315     MaxOffset = 63;
2316     break;
2317   case AArch64::PRFMui:
2318   case AArch64::LDRXui:
2319   case AArch64::LDRDui:
2320   case AArch64::STRXui:
2321   case AArch64::STRDui:
2322     Scale = TypeSize::Fixed(8);
2323     Width = 8;
2324     MinOffset = 0;
2325     MaxOffset = 4095;
2326     break;
2327   case AArch64::LDPWi:
2328   case AArch64::LDPSi:
2329   case AArch64::LDNPWi:
2330   case AArch64::LDNPSi:
2331   case AArch64::STPWi:
2332   case AArch64::STPSi:
2333   case AArch64::STNPWi:
2334   case AArch64::STNPSi:
2335     Scale = TypeSize::Fixed(4);
2336     Width = 8;
2337     MinOffset = -64;
2338     MaxOffset = 63;
2339     break;
2340   case AArch64::LDRWui:
2341   case AArch64::LDRSui:
2342   case AArch64::LDRSWui:
2343   case AArch64::STRWui:
2344   case AArch64::STRSui:
2345     Scale = TypeSize::Fixed(4);
2346     Width = 4;
2347     MinOffset = 0;
2348     MaxOffset = 4095;
2349     break;
2350   case AArch64::LDRHui:
2351   case AArch64::LDRHHui:
2352   case AArch64::LDRSHWui:
2353   case AArch64::LDRSHXui:
2354   case AArch64::STRHui:
2355   case AArch64::STRHHui:
2356     Scale = TypeSize::Fixed(2);
2357     Width = 2;
2358     MinOffset = 0;
2359     MaxOffset = 4095;
2360     break;
2361   case AArch64::LDRBui:
2362   case AArch64::LDRBBui:
2363   case AArch64::LDRSBWui:
2364   case AArch64::LDRSBXui:
2365   case AArch64::STRBui:
2366   case AArch64::STRBBui:
2367     Scale = TypeSize::Fixed(1);
2368     Width = 1;
2369     MinOffset = 0;
2370     MaxOffset = 4095;
2371     break;
2372   case AArch64::ADDG:
2373     Scale = TypeSize::Fixed(16);
2374     Width = 0;
2375     MinOffset = 0;
2376     MaxOffset = 63;
2377     break;
2378   case AArch64::TAGPstack:
2379     Scale = TypeSize::Fixed(16);
2380     Width = 0;
2381     // TAGP with a negative offset turns into SUBP, which has a maximum offset
2382     // of 63 (not 64!).
2383     MinOffset = -63;
2384     MaxOffset = 63;
2385     break;
2386   case AArch64::LDG:
2387   case AArch64::STGOffset:
2388   case AArch64::STZGOffset:
2389     Scale = TypeSize::Fixed(16);
2390     Width = 16;
2391     MinOffset = -256;
2392     MaxOffset = 255;
2393     break;
2394   case AArch64::STR_ZZZZXI:
2395   case AArch64::LDR_ZZZZXI:
2396     Scale = TypeSize::Scalable(16);
2397     Width = SVEMaxBytesPerVector * 4;
2398     MinOffset = -256;
2399     MaxOffset = 252;
2400     break;
2401   case AArch64::STR_ZZZXI:
2402   case AArch64::LDR_ZZZXI:
2403     Scale = TypeSize::Scalable(16);
2404     Width = SVEMaxBytesPerVector * 3;
2405     MinOffset = -256;
2406     MaxOffset = 253;
2407     break;
2408   case AArch64::STR_ZZXI:
2409   case AArch64::LDR_ZZXI:
2410     Scale = TypeSize::Scalable(16);
2411     Width = SVEMaxBytesPerVector * 2;
2412     MinOffset = -256;
2413     MaxOffset = 254;
2414     break;
2415   case AArch64::LDR_PXI:
2416   case AArch64::STR_PXI:
2417     Scale = TypeSize::Scalable(2);
2418     Width = SVEMaxBytesPerVector / 8;
2419     MinOffset = -256;
2420     MaxOffset = 255;
2421     break;
2422   case AArch64::LDR_ZXI:
2423   case AArch64::STR_ZXI:
2424     Scale = TypeSize::Scalable(16);
2425     Width = SVEMaxBytesPerVector;
2426     MinOffset = -256;
2427     MaxOffset = 255;
2428     break;
2429   case AArch64::LD1B_IMM:
2430   case AArch64::LD1H_IMM:
2431   case AArch64::LD1W_IMM:
2432   case AArch64::LD1D_IMM:
2433   case AArch64::ST1B_IMM:
2434   case AArch64::ST1H_IMM:
2435   case AArch64::ST1W_IMM:
2436   case AArch64::ST1D_IMM:
2437     // A full vectors worth of data
2438     // Width = mbytes * elements
2439     Scale = TypeSize::Scalable(16);
2440     Width = SVEMaxBytesPerVector;
2441     MinOffset = -8;
2442     MaxOffset = 7;
2443     break;
2444   case AArch64::LD1B_H_IMM:
2445   case AArch64::LD1SB_H_IMM:
2446   case AArch64::LD1H_S_IMM:
2447   case AArch64::LD1SH_S_IMM:
2448   case AArch64::LD1W_D_IMM:
2449   case AArch64::LD1SW_D_IMM:
2450   case AArch64::ST1B_H_IMM:
2451   case AArch64::ST1H_S_IMM:
2452   case AArch64::ST1W_D_IMM:
2453     // A half vector worth of data
2454     // Width = mbytes * elements
2455     Scale = TypeSize::Scalable(8);
2456     Width = SVEMaxBytesPerVector / 2;
2457     MinOffset = -8;
2458     MaxOffset = 7;
2459     break;
2460   case AArch64::LD1B_S_IMM:
2461   case AArch64::LD1SB_S_IMM:
2462   case AArch64::LD1H_D_IMM:
2463   case AArch64::LD1SH_D_IMM:
2464   case AArch64::ST1B_S_IMM:
2465   case AArch64::ST1H_D_IMM:
2466     // A quarter vector worth of data
2467     // Width = mbytes * elements
2468     Scale = TypeSize::Scalable(4);
2469     Width = SVEMaxBytesPerVector / 4;
2470     MinOffset = -8;
2471     MaxOffset = 7;
2472     break;
2473   case AArch64::LD1B_D_IMM:
2474   case AArch64::LD1SB_D_IMM:
2475   case AArch64::ST1B_D_IMM:
2476     // A eighth vector worth of data
2477     // Width = mbytes * elements
2478     Scale = TypeSize::Scalable(2);
2479     Width = SVEMaxBytesPerVector / 8;
2480     MinOffset = -8;
2481     MaxOffset = 7;
2482     break;
2483   case AArch64::ST2GOffset:
2484   case AArch64::STZ2GOffset:
2485     Scale = TypeSize::Fixed(16);
2486     Width = 32;
2487     MinOffset = -256;
2488     MaxOffset = 255;
2489     break;
2490   case AArch64::STGPi:
2491     Scale = TypeSize::Fixed(16);
2492     Width = 16;
2493     MinOffset = -64;
2494     MaxOffset = 63;
2495     break;
2496   }
2497 
2498   return true;
2499 }
2500 
2501 // Scaling factor for unscaled load or store.
2502 int AArch64InstrInfo::getMemScale(unsigned Opc) {
2503   switch (Opc) {
2504   default:
2505     llvm_unreachable("Opcode has unknown scale!");
2506   case AArch64::LDRBBui:
2507   case AArch64::LDURBBi:
2508   case AArch64::LDRSBWui:
2509   case AArch64::LDURSBWi:
2510   case AArch64::STRBBui:
2511   case AArch64::STURBBi:
2512     return 1;
2513   case AArch64::LDRHHui:
2514   case AArch64::LDURHHi:
2515   case AArch64::LDRSHWui:
2516   case AArch64::LDURSHWi:
2517   case AArch64::STRHHui:
2518   case AArch64::STURHHi:
2519     return 2;
2520   case AArch64::LDRSui:
2521   case AArch64::LDURSi:
2522   case AArch64::LDRSWui:
2523   case AArch64::LDURSWi:
2524   case AArch64::LDRWui:
2525   case AArch64::LDURWi:
2526   case AArch64::STRSui:
2527   case AArch64::STURSi:
2528   case AArch64::STRWui:
2529   case AArch64::STURWi:
2530   case AArch64::LDPSi:
2531   case AArch64::LDPSWi:
2532   case AArch64::LDPWi:
2533   case AArch64::STPSi:
2534   case AArch64::STPWi:
2535     return 4;
2536   case AArch64::LDRDui:
2537   case AArch64::LDURDi:
2538   case AArch64::LDRXui:
2539   case AArch64::LDURXi:
2540   case AArch64::STRDui:
2541   case AArch64::STURDi:
2542   case AArch64::STRXui:
2543   case AArch64::STURXi:
2544   case AArch64::LDPDi:
2545   case AArch64::LDPXi:
2546   case AArch64::STPDi:
2547   case AArch64::STPXi:
2548     return 8;
2549   case AArch64::LDRQui:
2550   case AArch64::LDURQi:
2551   case AArch64::STRQui:
2552   case AArch64::STURQi:
2553   case AArch64::LDPQi:
2554   case AArch64::STPQi:
2555   case AArch64::STGOffset:
2556   case AArch64::STZGOffset:
2557   case AArch64::ST2GOffset:
2558   case AArch64::STZ2GOffset:
2559   case AArch64::STGPi:
2560     return 16;
2561   }
2562 }
2563 
2564 // Scale the unscaled offsets.  Returns false if the unscaled offset can't be
2565 // scaled.
2566 static bool scaleOffset(unsigned Opc, int64_t &Offset) {
2567   int Scale = AArch64InstrInfo::getMemScale(Opc);
2568 
2569   // If the byte-offset isn't a multiple of the stride, we can't scale this
2570   // offset.
2571   if (Offset % Scale != 0)
2572     return false;
2573 
2574   // Convert the byte-offset used by unscaled into an "element" offset used
2575   // by the scaled pair load/store instructions.
2576   Offset /= Scale;
2577   return true;
2578 }
2579 
2580 static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
2581   if (FirstOpc == SecondOpc)
2582     return true;
2583   // We can also pair sign-ext and zero-ext instructions.
2584   switch (FirstOpc) {
2585   default:
2586     return false;
2587   case AArch64::LDRWui:
2588   case AArch64::LDURWi:
2589     return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
2590   case AArch64::LDRSWui:
2591   case AArch64::LDURSWi:
2592     return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
2593   }
2594   // These instructions can't be paired based on their opcodes.
2595   return false;
2596 }
2597 
2598 static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
2599                             int64_t Offset1, unsigned Opcode1, int FI2,
2600                             int64_t Offset2, unsigned Opcode2) {
2601   // Accesses through fixed stack object frame indices may access a different
2602   // fixed stack slot. Check that the object offsets + offsets match.
2603   if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
2604     int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
2605     int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
2606     assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
2607     // Convert to scaled object offsets.
2608     int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
2609     if (ObjectOffset1 % Scale1 != 0)
2610       return false;
2611     ObjectOffset1 /= Scale1;
2612     int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
2613     if (ObjectOffset2 % Scale2 != 0)
2614       return false;
2615     ObjectOffset2 /= Scale2;
2616     ObjectOffset1 += Offset1;
2617     ObjectOffset2 += Offset2;
2618     return ObjectOffset1 + 1 == ObjectOffset2;
2619   }
2620 
2621   return FI1 == FI2;
2622 }
2623 
2624 /// Detect opportunities for ldp/stp formation.
2625 ///
2626 /// Only called for LdSt for which getMemOperandWithOffset returns true.
2627 bool AArch64InstrInfo::shouldClusterMemOps(
2628     ArrayRef<const MachineOperand *> BaseOps1,
2629     ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2630     unsigned NumBytes) const {
2631   assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2632   const MachineOperand &BaseOp1 = *BaseOps1.front();
2633   const MachineOperand &BaseOp2 = *BaseOps2.front();
2634   const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2635   const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2636   if (BaseOp1.getType() != BaseOp2.getType())
2637     return false;
2638 
2639   assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2640          "Only base registers and frame indices are supported.");
2641 
2642   // Check for both base regs and base FI.
2643   if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
2644     return false;
2645 
2646   // Only cluster up to a single pair.
2647   if (NumLoads > 2)
2648     return false;
2649 
2650   if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
2651     return false;
2652 
2653   // Can we pair these instructions based on their opcodes?
2654   unsigned FirstOpc = FirstLdSt.getOpcode();
2655   unsigned SecondOpc = SecondLdSt.getOpcode();
2656   if (!canPairLdStOpc(FirstOpc, SecondOpc))
2657     return false;
2658 
2659   // Can't merge volatiles or load/stores that have a hint to avoid pair
2660   // formation, for example.
2661   if (!isCandidateToMergeOrPair(FirstLdSt) ||
2662       !isCandidateToMergeOrPair(SecondLdSt))
2663     return false;
2664 
2665   // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
2666   int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
2667   if (isUnscaledLdSt(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
2668     return false;
2669 
2670   int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
2671   if (isUnscaledLdSt(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
2672     return false;
2673 
2674   // Pairwise instructions have a 7-bit signed offset field.
2675   if (Offset1 > 63 || Offset1 < -64)
2676     return false;
2677 
2678   // The caller should already have ordered First/SecondLdSt by offset.
2679   // Note: except for non-equal frame index bases
2680   if (BaseOp1.isFI()) {
2681     assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
2682            "Caller should have ordered offsets.");
2683 
2684     const MachineFrameInfo &MFI =
2685         FirstLdSt.getParent()->getParent()->getFrameInfo();
2686     return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
2687                            BaseOp2.getIndex(), Offset2, SecondOpc);
2688   }
2689 
2690   assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2691 
2692   return Offset1 + 1 == Offset2;
2693 }
2694 
2695 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
2696                                             unsigned Reg, unsigned SubIdx,
2697                                             unsigned State,
2698                                             const TargetRegisterInfo *TRI) {
2699   if (!SubIdx)
2700     return MIB.addReg(Reg, State);
2701 
2702   if (Register::isPhysicalRegister(Reg))
2703     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
2704   return MIB.addReg(Reg, State, SubIdx);
2705 }
2706 
2707 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
2708                                         unsigned NumRegs) {
2709   // We really want the positive remainder mod 32 here, that happens to be
2710   // easily obtainable with a mask.
2711   return ((DestReg - SrcReg) & 0x1f) < NumRegs;
2712 }
2713 
2714 void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
2715                                         MachineBasicBlock::iterator I,
2716                                         const DebugLoc &DL, MCRegister DestReg,
2717                                         MCRegister SrcReg, bool KillSrc,
2718                                         unsigned Opcode,
2719                                         ArrayRef<unsigned> Indices) const {
2720   assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
2721   const TargetRegisterInfo *TRI = &getRegisterInfo();
2722   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2723   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2724   unsigned NumRegs = Indices.size();
2725 
2726   int SubReg = 0, End = NumRegs, Incr = 1;
2727   if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
2728     SubReg = NumRegs - 1;
2729     End = -1;
2730     Incr = -1;
2731   }
2732 
2733   for (; SubReg != End; SubReg += Incr) {
2734     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2735     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2736     AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
2737     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2738   }
2739 }
2740 
2741 void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
2742                                        MachineBasicBlock::iterator I,
2743                                        DebugLoc DL, unsigned DestReg,
2744                                        unsigned SrcReg, bool KillSrc,
2745                                        unsigned Opcode, unsigned ZeroReg,
2746                                        llvm::ArrayRef<unsigned> Indices) const {
2747   const TargetRegisterInfo *TRI = &getRegisterInfo();
2748   unsigned NumRegs = Indices.size();
2749 
2750 #ifndef NDEBUG
2751   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2752   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2753   assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
2754          "GPR reg sequences should not be able to overlap");
2755 #endif
2756 
2757   for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
2758     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2759     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2760     MIB.addReg(ZeroReg);
2761     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2762     MIB.addImm(0);
2763   }
2764 }
2765 
2766 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
2767                                    MachineBasicBlock::iterator I,
2768                                    const DebugLoc &DL, MCRegister DestReg,
2769                                    MCRegister SrcReg, bool KillSrc) const {
2770   if (AArch64::GPR32spRegClass.contains(DestReg) &&
2771       (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
2772     const TargetRegisterInfo *TRI = &getRegisterInfo();
2773 
2774     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
2775       // If either operand is WSP, expand to ADD #0.
2776       if (Subtarget.hasZeroCycleRegMove()) {
2777         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
2778         MCRegister DestRegX = TRI->getMatchingSuperReg(
2779             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2780         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2781             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2782         // This instruction is reading and writing X registers.  This may upset
2783         // the register scavenger and machine verifier, so we need to indicate
2784         // that we are reading an undefined value from SrcRegX, but a proper
2785         // value from SrcReg.
2786         BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
2787             .addReg(SrcRegX, RegState::Undef)
2788             .addImm(0)
2789             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2790             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2791       } else {
2792         BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
2793             .addReg(SrcReg, getKillRegState(KillSrc))
2794             .addImm(0)
2795             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2796       }
2797     } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
2798       BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
2799           .addImm(0)
2800           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2801     } else {
2802       if (Subtarget.hasZeroCycleRegMove()) {
2803         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
2804         MCRegister DestRegX = TRI->getMatchingSuperReg(
2805             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2806         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2807             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2808         // This instruction is reading and writing X registers.  This may upset
2809         // the register scavenger and machine verifier, so we need to indicate
2810         // that we are reading an undefined value from SrcRegX, but a proper
2811         // value from SrcReg.
2812         BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
2813             .addReg(AArch64::XZR)
2814             .addReg(SrcRegX, RegState::Undef)
2815             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2816       } else {
2817         // Otherwise, expand to ORR WZR.
2818         BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
2819             .addReg(AArch64::WZR)
2820             .addReg(SrcReg, getKillRegState(KillSrc));
2821       }
2822     }
2823     return;
2824   }
2825 
2826   // Copy a Predicate register by ORRing with itself.
2827   if (AArch64::PPRRegClass.contains(DestReg) &&
2828       AArch64::PPRRegClass.contains(SrcReg)) {
2829     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2830     BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
2831       .addReg(SrcReg) // Pg
2832       .addReg(SrcReg)
2833       .addReg(SrcReg, getKillRegState(KillSrc));
2834     return;
2835   }
2836 
2837   // Copy a Z register by ORRing with itself.
2838   if (AArch64::ZPRRegClass.contains(DestReg) &&
2839       AArch64::ZPRRegClass.contains(SrcReg)) {
2840     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2841     BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
2842       .addReg(SrcReg)
2843       .addReg(SrcReg, getKillRegState(KillSrc));
2844     return;
2845   }
2846 
2847   // Copy a Z register pair by copying the individual sub-registers.
2848   if (AArch64::ZPR2RegClass.contains(DestReg) &&
2849       AArch64::ZPR2RegClass.contains(SrcReg)) {
2850     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
2851     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2852                      Indices);
2853     return;
2854   }
2855 
2856   // Copy a Z register triple by copying the individual sub-registers.
2857   if (AArch64::ZPR3RegClass.contains(DestReg) &&
2858       AArch64::ZPR3RegClass.contains(SrcReg)) {
2859     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
2860                                        AArch64::zsub2};
2861     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2862                      Indices);
2863     return;
2864   }
2865 
2866   // Copy a Z register quad by copying the individual sub-registers.
2867   if (AArch64::ZPR4RegClass.contains(DestReg) &&
2868       AArch64::ZPR4RegClass.contains(SrcReg)) {
2869     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
2870                                        AArch64::zsub2, AArch64::zsub3};
2871     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2872                      Indices);
2873     return;
2874   }
2875 
2876   if (AArch64::GPR64spRegClass.contains(DestReg) &&
2877       (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
2878     if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
2879       // If either operand is SP, expand to ADD #0.
2880       BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
2881           .addReg(SrcReg, getKillRegState(KillSrc))
2882           .addImm(0)
2883           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2884     } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
2885       BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
2886           .addImm(0)
2887           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2888     } else {
2889       // Otherwise, expand to ORR XZR.
2890       BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
2891           .addReg(AArch64::XZR)
2892           .addReg(SrcReg, getKillRegState(KillSrc));
2893     }
2894     return;
2895   }
2896 
2897   // Copy a DDDD register quad by copying the individual sub-registers.
2898   if (AArch64::DDDDRegClass.contains(DestReg) &&
2899       AArch64::DDDDRegClass.contains(SrcReg)) {
2900     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2901                                        AArch64::dsub2, AArch64::dsub3};
2902     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2903                      Indices);
2904     return;
2905   }
2906 
2907   // Copy a DDD register triple by copying the individual sub-registers.
2908   if (AArch64::DDDRegClass.contains(DestReg) &&
2909       AArch64::DDDRegClass.contains(SrcReg)) {
2910     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2911                                        AArch64::dsub2};
2912     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2913                      Indices);
2914     return;
2915   }
2916 
2917   // Copy a DD register pair by copying the individual sub-registers.
2918   if (AArch64::DDRegClass.contains(DestReg) &&
2919       AArch64::DDRegClass.contains(SrcReg)) {
2920     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
2921     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2922                      Indices);
2923     return;
2924   }
2925 
2926   // Copy a QQQQ register quad by copying the individual sub-registers.
2927   if (AArch64::QQQQRegClass.contains(DestReg) &&
2928       AArch64::QQQQRegClass.contains(SrcReg)) {
2929     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2930                                        AArch64::qsub2, AArch64::qsub3};
2931     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2932                      Indices);
2933     return;
2934   }
2935 
2936   // Copy a QQQ register triple by copying the individual sub-registers.
2937   if (AArch64::QQQRegClass.contains(DestReg) &&
2938       AArch64::QQQRegClass.contains(SrcReg)) {
2939     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2940                                        AArch64::qsub2};
2941     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2942                      Indices);
2943     return;
2944   }
2945 
2946   // Copy a QQ register pair by copying the individual sub-registers.
2947   if (AArch64::QQRegClass.contains(DestReg) &&
2948       AArch64::QQRegClass.contains(SrcReg)) {
2949     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
2950     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2951                      Indices);
2952     return;
2953   }
2954 
2955   if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
2956       AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
2957     static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
2958     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
2959                     AArch64::XZR, Indices);
2960     return;
2961   }
2962 
2963   if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
2964       AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
2965     static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
2966     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
2967                     AArch64::WZR, Indices);
2968     return;
2969   }
2970 
2971   if (AArch64::FPR128RegClass.contains(DestReg) &&
2972       AArch64::FPR128RegClass.contains(SrcReg)) {
2973     if (Subtarget.hasNEON()) {
2974       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2975           .addReg(SrcReg)
2976           .addReg(SrcReg, getKillRegState(KillSrc));
2977     } else {
2978       BuildMI(MBB, I, DL, get(AArch64::STRQpre))
2979           .addReg(AArch64::SP, RegState::Define)
2980           .addReg(SrcReg, getKillRegState(KillSrc))
2981           .addReg(AArch64::SP)
2982           .addImm(-16);
2983       BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
2984           .addReg(AArch64::SP, RegState::Define)
2985           .addReg(DestReg, RegState::Define)
2986           .addReg(AArch64::SP)
2987           .addImm(16);
2988     }
2989     return;
2990   }
2991 
2992   if (AArch64::FPR64RegClass.contains(DestReg) &&
2993       AArch64::FPR64RegClass.contains(SrcReg)) {
2994     if (Subtarget.hasNEON()) {
2995       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
2996                                        &AArch64::FPR128RegClass);
2997       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
2998                                       &AArch64::FPR128RegClass);
2999       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
3000           .addReg(SrcReg)
3001           .addReg(SrcReg, getKillRegState(KillSrc));
3002     } else {
3003       BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
3004           .addReg(SrcReg, getKillRegState(KillSrc));
3005     }
3006     return;
3007   }
3008 
3009   if (AArch64::FPR32RegClass.contains(DestReg) &&
3010       AArch64::FPR32RegClass.contains(SrcReg)) {
3011     if (Subtarget.hasNEON()) {
3012       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
3013                                        &AArch64::FPR128RegClass);
3014       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
3015                                       &AArch64::FPR128RegClass);
3016       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
3017           .addReg(SrcReg)
3018           .addReg(SrcReg, getKillRegState(KillSrc));
3019     } else {
3020       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3021           .addReg(SrcReg, getKillRegState(KillSrc));
3022     }
3023     return;
3024   }
3025 
3026   if (AArch64::FPR16RegClass.contains(DestReg) &&
3027       AArch64::FPR16RegClass.contains(SrcReg)) {
3028     if (Subtarget.hasNEON()) {
3029       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
3030                                        &AArch64::FPR128RegClass);
3031       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
3032                                       &AArch64::FPR128RegClass);
3033       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
3034           .addReg(SrcReg)
3035           .addReg(SrcReg, getKillRegState(KillSrc));
3036     } else {
3037       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
3038                                        &AArch64::FPR32RegClass);
3039       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
3040                                       &AArch64::FPR32RegClass);
3041       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3042           .addReg(SrcReg, getKillRegState(KillSrc));
3043     }
3044     return;
3045   }
3046 
3047   if (AArch64::FPR8RegClass.contains(DestReg) &&
3048       AArch64::FPR8RegClass.contains(SrcReg)) {
3049     if (Subtarget.hasNEON()) {
3050       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
3051                                        &AArch64::FPR128RegClass);
3052       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
3053                                       &AArch64::FPR128RegClass);
3054       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
3055           .addReg(SrcReg)
3056           .addReg(SrcReg, getKillRegState(KillSrc));
3057     } else {
3058       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
3059                                        &AArch64::FPR32RegClass);
3060       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
3061                                       &AArch64::FPR32RegClass);
3062       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3063           .addReg(SrcReg, getKillRegState(KillSrc));
3064     }
3065     return;
3066   }
3067 
3068   // Copies between GPR64 and FPR64.
3069   if (AArch64::FPR64RegClass.contains(DestReg) &&
3070       AArch64::GPR64RegClass.contains(SrcReg)) {
3071     BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
3072         .addReg(SrcReg, getKillRegState(KillSrc));
3073     return;
3074   }
3075   if (AArch64::GPR64RegClass.contains(DestReg) &&
3076       AArch64::FPR64RegClass.contains(SrcReg)) {
3077     BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
3078         .addReg(SrcReg, getKillRegState(KillSrc));
3079     return;
3080   }
3081   // Copies between GPR32 and FPR32.
3082   if (AArch64::FPR32RegClass.contains(DestReg) &&
3083       AArch64::GPR32RegClass.contains(SrcReg)) {
3084     BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
3085         .addReg(SrcReg, getKillRegState(KillSrc));
3086     return;
3087   }
3088   if (AArch64::GPR32RegClass.contains(DestReg) &&
3089       AArch64::FPR32RegClass.contains(SrcReg)) {
3090     BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
3091         .addReg(SrcReg, getKillRegState(KillSrc));
3092     return;
3093   }
3094 
3095   if (DestReg == AArch64::NZCV) {
3096     assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
3097     BuildMI(MBB, I, DL, get(AArch64::MSR))
3098         .addImm(AArch64SysReg::NZCV)
3099         .addReg(SrcReg, getKillRegState(KillSrc))
3100         .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
3101     return;
3102   }
3103 
3104   if (SrcReg == AArch64::NZCV) {
3105     assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
3106     BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
3107         .addImm(AArch64SysReg::NZCV)
3108         .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
3109     return;
3110   }
3111 
3112   llvm_unreachable("unimplemented reg-to-reg copy");
3113 }
3114 
3115 static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
3116                                     MachineBasicBlock &MBB,
3117                                     MachineBasicBlock::iterator InsertBefore,
3118                                     const MCInstrDesc &MCID,
3119                                     Register SrcReg, bool IsKill,
3120                                     unsigned SubIdx0, unsigned SubIdx1, int FI,
3121                                     MachineMemOperand *MMO) {
3122   Register SrcReg0 = SrcReg;
3123   Register SrcReg1 = SrcReg;
3124   if (Register::isPhysicalRegister(SrcReg)) {
3125     SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
3126     SubIdx0 = 0;
3127     SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
3128     SubIdx1 = 0;
3129   }
3130   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3131       .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
3132       .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
3133       .addFrameIndex(FI)
3134       .addImm(0)
3135       .addMemOperand(MMO);
3136 }
3137 
3138 void AArch64InstrInfo::storeRegToStackSlot(
3139     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
3140     bool isKill, int FI, const TargetRegisterClass *RC,
3141     const TargetRegisterInfo *TRI) const {
3142   MachineFunction &MF = *MBB.getParent();
3143   MachineFrameInfo &MFI = MF.getFrameInfo();
3144 
3145   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3146   MachineMemOperand *MMO =
3147       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
3148                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3149   unsigned Opc = 0;
3150   bool Offset = true;
3151   unsigned StackID = TargetStackID::Default;
3152   switch (TRI->getSpillSize(*RC)) {
3153   case 1:
3154     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3155       Opc = AArch64::STRBui;
3156     break;
3157   case 2:
3158     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3159       Opc = AArch64::STRHui;
3160     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3161       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3162       Opc = AArch64::STR_PXI;
3163       StackID = TargetStackID::SVEVector;
3164     }
3165     break;
3166   case 4:
3167     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3168       Opc = AArch64::STRWui;
3169       if (Register::isVirtualRegister(SrcReg))
3170         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
3171       else
3172         assert(SrcReg != AArch64::WSP);
3173     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3174       Opc = AArch64::STRSui;
3175     break;
3176   case 8:
3177     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3178       Opc = AArch64::STRXui;
3179       if (Register::isVirtualRegister(SrcReg))
3180         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3181       else
3182         assert(SrcReg != AArch64::SP);
3183     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3184       Opc = AArch64::STRDui;
3185     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3186       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3187                               get(AArch64::STPWi), SrcReg, isKill,
3188                               AArch64::sube32, AArch64::subo32, FI, MMO);
3189       return;
3190     }
3191     break;
3192   case 16:
3193     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3194       Opc = AArch64::STRQui;
3195     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3196       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3197       Opc = AArch64::ST1Twov1d;
3198       Offset = false;
3199     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3200       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3201                               get(AArch64::STPXi), SrcReg, isKill,
3202                               AArch64::sube64, AArch64::subo64, FI, MMO);
3203       return;
3204     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3205       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3206       Opc = AArch64::STR_ZXI;
3207       StackID = TargetStackID::SVEVector;
3208     }
3209     break;
3210   case 24:
3211     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3212       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3213       Opc = AArch64::ST1Threev1d;
3214       Offset = false;
3215     }
3216     break;
3217   case 32:
3218     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3219       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3220       Opc = AArch64::ST1Fourv1d;
3221       Offset = false;
3222     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3223       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3224       Opc = AArch64::ST1Twov2d;
3225       Offset = false;
3226     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3227       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3228       Opc = AArch64::STR_ZZXI;
3229       StackID = TargetStackID::SVEVector;
3230     }
3231     break;
3232   case 48:
3233     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3234       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3235       Opc = AArch64::ST1Threev2d;
3236       Offset = false;
3237     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3238       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3239       Opc = AArch64::STR_ZZZXI;
3240       StackID = TargetStackID::SVEVector;
3241     }
3242     break;
3243   case 64:
3244     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3245       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3246       Opc = AArch64::ST1Fourv2d;
3247       Offset = false;
3248     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3249       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3250       Opc = AArch64::STR_ZZZZXI;
3251       StackID = TargetStackID::SVEVector;
3252     }
3253     break;
3254   }
3255   assert(Opc && "Unknown register class");
3256   MFI.setStackID(FI, StackID);
3257 
3258   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3259                                      .addReg(SrcReg, getKillRegState(isKill))
3260                                      .addFrameIndex(FI);
3261 
3262   if (Offset)
3263     MI.addImm(0);
3264   MI.addMemOperand(MMO);
3265 }
3266 
3267 static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
3268                                      MachineBasicBlock &MBB,
3269                                      MachineBasicBlock::iterator InsertBefore,
3270                                      const MCInstrDesc &MCID,
3271                                      Register DestReg, unsigned SubIdx0,
3272                                      unsigned SubIdx1, int FI,
3273                                      MachineMemOperand *MMO) {
3274   Register DestReg0 = DestReg;
3275   Register DestReg1 = DestReg;
3276   bool IsUndef = true;
3277   if (Register::isPhysicalRegister(DestReg)) {
3278     DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
3279     SubIdx0 = 0;
3280     DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
3281     SubIdx1 = 0;
3282     IsUndef = false;
3283   }
3284   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3285       .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
3286       .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
3287       .addFrameIndex(FI)
3288       .addImm(0)
3289       .addMemOperand(MMO);
3290 }
3291 
3292 void AArch64InstrInfo::loadRegFromStackSlot(
3293     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
3294     int FI, const TargetRegisterClass *RC,
3295     const TargetRegisterInfo *TRI) const {
3296   MachineFunction &MF = *MBB.getParent();
3297   MachineFrameInfo &MFI = MF.getFrameInfo();
3298   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3299   MachineMemOperand *MMO =
3300       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
3301                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3302 
3303   unsigned Opc = 0;
3304   bool Offset = true;
3305   unsigned StackID = TargetStackID::Default;
3306   switch (TRI->getSpillSize(*RC)) {
3307   case 1:
3308     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3309       Opc = AArch64::LDRBui;
3310     break;
3311   case 2:
3312     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3313       Opc = AArch64::LDRHui;
3314     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3315       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3316       Opc = AArch64::LDR_PXI;
3317       StackID = TargetStackID::SVEVector;
3318     }
3319     break;
3320   case 4:
3321     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3322       Opc = AArch64::LDRWui;
3323       if (Register::isVirtualRegister(DestReg))
3324         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
3325       else
3326         assert(DestReg != AArch64::WSP);
3327     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3328       Opc = AArch64::LDRSui;
3329     break;
3330   case 8:
3331     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3332       Opc = AArch64::LDRXui;
3333       if (Register::isVirtualRegister(DestReg))
3334         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
3335       else
3336         assert(DestReg != AArch64::SP);
3337     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3338       Opc = AArch64::LDRDui;
3339     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3340       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3341                                get(AArch64::LDPWi), DestReg, AArch64::sube32,
3342                                AArch64::subo32, FI, MMO);
3343       return;
3344     }
3345     break;
3346   case 16:
3347     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3348       Opc = AArch64::LDRQui;
3349     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3350       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3351       Opc = AArch64::LD1Twov1d;
3352       Offset = false;
3353     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3354       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3355                                get(AArch64::LDPXi), DestReg, AArch64::sube64,
3356                                AArch64::subo64, FI, MMO);
3357       return;
3358     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3359       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3360       Opc = AArch64::LDR_ZXI;
3361       StackID = TargetStackID::SVEVector;
3362     }
3363     break;
3364   case 24:
3365     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3366       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3367       Opc = AArch64::LD1Threev1d;
3368       Offset = false;
3369     }
3370     break;
3371   case 32:
3372     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3373       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3374       Opc = AArch64::LD1Fourv1d;
3375       Offset = false;
3376     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3377       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3378       Opc = AArch64::LD1Twov2d;
3379       Offset = false;
3380     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3381       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3382       Opc = AArch64::LDR_ZZXI;
3383       StackID = TargetStackID::SVEVector;
3384     }
3385     break;
3386   case 48:
3387     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3388       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3389       Opc = AArch64::LD1Threev2d;
3390       Offset = false;
3391     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3392       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3393       Opc = AArch64::LDR_ZZZXI;
3394       StackID = TargetStackID::SVEVector;
3395     }
3396     break;
3397   case 64:
3398     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3399       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3400       Opc = AArch64::LD1Fourv2d;
3401       Offset = false;
3402     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3403       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3404       Opc = AArch64::LDR_ZZZZXI;
3405       StackID = TargetStackID::SVEVector;
3406     }
3407     break;
3408   }
3409 
3410   assert(Opc && "Unknown register class");
3411   MFI.setStackID(FI, StackID);
3412 
3413   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3414                                      .addReg(DestReg, getDefRegState(true))
3415                                      .addFrameIndex(FI);
3416   if (Offset)
3417     MI.addImm(0);
3418   MI.addMemOperand(MMO);
3419 }
3420 
3421 bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
3422                                            const MachineInstr &UseMI,
3423                                            const TargetRegisterInfo *TRI) {
3424   return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
3425                                          UseMI.getIterator()),
3426                 [TRI](const MachineInstr &I) {
3427                   return I.modifiesRegister(AArch64::NZCV, TRI) ||
3428                          I.readsRegister(AArch64::NZCV, TRI);
3429                 });
3430 }
3431 
3432 // Helper function to emit a frame offset adjustment from a given
3433 // pointer (SrcReg), stored into DestReg. This function is explicit
3434 // in that it requires the opcode.
3435 static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
3436                                MachineBasicBlock::iterator MBBI,
3437                                const DebugLoc &DL, unsigned DestReg,
3438                                unsigned SrcReg, int64_t Offset, unsigned Opc,
3439                                const TargetInstrInfo *TII,
3440                                MachineInstr::MIFlag Flag, bool NeedsWinCFI,
3441                                bool *HasWinCFI) {
3442   int Sign = 1;
3443   unsigned MaxEncoding, ShiftSize;
3444   switch (Opc) {
3445   case AArch64::ADDXri:
3446   case AArch64::ADDSXri:
3447   case AArch64::SUBXri:
3448   case AArch64::SUBSXri:
3449     MaxEncoding = 0xfff;
3450     ShiftSize = 12;
3451     break;
3452   case AArch64::ADDVL_XXI:
3453   case AArch64::ADDPL_XXI:
3454     MaxEncoding = 31;
3455     ShiftSize = 0;
3456     if (Offset < 0) {
3457       MaxEncoding = 32;
3458       Sign = -1;
3459       Offset = -Offset;
3460     }
3461     break;
3462   default:
3463     llvm_unreachable("Unsupported opcode");
3464   }
3465 
3466   // FIXME: If the offset won't fit in 24-bits, compute the offset into a
3467   // scratch register.  If DestReg is a virtual register, use it as the
3468   // scratch register; otherwise, create a new virtual register (to be
3469   // replaced by the scavenger at the end of PEI).  That case can be optimized
3470   // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
3471   // register can be loaded with offset%8 and the add/sub can use an extending
3472   // instruction with LSL#3.
3473   // Currently the function handles any offsets but generates a poor sequence
3474   // of code.
3475   //  assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
3476 
3477   const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
3478   Register TmpReg = DestReg;
3479   if (TmpReg == AArch64::XZR)
3480     TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
3481         &AArch64::GPR64RegClass);
3482   do {
3483     uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
3484     unsigned LocalShiftSize = 0;
3485     if (ThisVal > MaxEncoding) {
3486       ThisVal = ThisVal >> ShiftSize;
3487       LocalShiftSize = ShiftSize;
3488     }
3489     assert((ThisVal >> ShiftSize) <= MaxEncoding &&
3490            "Encoding cannot handle value that big");
3491 
3492     Offset -= ThisVal << LocalShiftSize;
3493     if (Offset == 0)
3494       TmpReg = DestReg;
3495     auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg)
3496                    .addReg(SrcReg)
3497                    .addImm(Sign * (int)ThisVal);
3498     if (ShiftSize)
3499       MBI = MBI.addImm(
3500           AArch64_AM::getShifterImm(AArch64_AM::LSL, LocalShiftSize));
3501     MBI = MBI.setMIFlag(Flag);
3502 
3503     if (NeedsWinCFI) {
3504       assert(Sign == 1 && "SEH directives should always have a positive sign");
3505       int Imm = (int)(ThisVal << LocalShiftSize);
3506       if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
3507           (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
3508         if (HasWinCFI)
3509           *HasWinCFI = true;
3510         if (Imm == 0)
3511           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
3512         else
3513           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
3514               .addImm(Imm)
3515               .setMIFlag(Flag);
3516         assert(Offset == 0 && "Expected remaining offset to be zero to "
3517                               "emit a single SEH directive");
3518       } else if (DestReg == AArch64::SP) {
3519         if (HasWinCFI)
3520           *HasWinCFI = true;
3521         assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
3522         BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
3523             .addImm(Imm)
3524             .setMIFlag(Flag);
3525       }
3526       if (HasWinCFI)
3527         *HasWinCFI = true;
3528     }
3529 
3530     SrcReg = TmpReg;
3531   } while (Offset);
3532 }
3533 
3534 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
3535                            MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
3536                            unsigned DestReg, unsigned SrcReg,
3537                            StackOffset Offset, const TargetInstrInfo *TII,
3538                            MachineInstr::MIFlag Flag, bool SetNZCV,
3539                            bool NeedsWinCFI, bool *HasWinCFI) {
3540   int64_t Bytes, NumPredicateVectors, NumDataVectors;
3541   Offset.getForFrameOffset(Bytes, NumPredicateVectors, NumDataVectors);
3542 
3543   // First emit non-scalable frame offsets, or a simple 'mov'.
3544   if (Bytes || (!Offset && SrcReg != DestReg)) {
3545     assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
3546            "SP increment/decrement not 8-byte aligned");
3547     unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
3548     if (Bytes < 0) {
3549       Bytes = -Bytes;
3550       Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
3551     }
3552     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
3553                        NeedsWinCFI, HasWinCFI);
3554     SrcReg = DestReg;
3555   }
3556 
3557   assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
3558          "SetNZCV not supported with SVE vectors");
3559   assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
3560          "WinCFI not supported with SVE vectors");
3561 
3562   if (NumDataVectors) {
3563     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
3564                        AArch64::ADDVL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3565     SrcReg = DestReg;
3566   }
3567 
3568   if (NumPredicateVectors) {
3569     assert(DestReg != AArch64::SP && "Unaligned access to SP");
3570     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
3571                        AArch64::ADDPL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3572   }
3573 }
3574 
3575 MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
3576     MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
3577     MachineBasicBlock::iterator InsertPt, int FrameIndex,
3578     LiveIntervals *LIS, VirtRegMap *VRM) const {
3579   // This is a bit of a hack. Consider this instruction:
3580   //
3581   //   %0 = COPY %sp; GPR64all:%0
3582   //
3583   // We explicitly chose GPR64all for the virtual register so such a copy might
3584   // be eliminated by RegisterCoalescer. However, that may not be possible, and
3585   // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
3586   // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
3587   //
3588   // To prevent that, we are going to constrain the %0 register class here.
3589   //
3590   // <rdar://problem/11522048>
3591   //
3592   if (MI.isFullCopy()) {
3593     Register DstReg = MI.getOperand(0).getReg();
3594     Register SrcReg = MI.getOperand(1).getReg();
3595     if (SrcReg == AArch64::SP && Register::isVirtualRegister(DstReg)) {
3596       MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
3597       return nullptr;
3598     }
3599     if (DstReg == AArch64::SP && Register::isVirtualRegister(SrcReg)) {
3600       MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3601       return nullptr;
3602     }
3603   }
3604 
3605   // Handle the case where a copy is being spilled or filled but the source
3606   // and destination register class don't match.  For example:
3607   //
3608   //   %0 = COPY %xzr; GPR64common:%0
3609   //
3610   // In this case we can still safely fold away the COPY and generate the
3611   // following spill code:
3612   //
3613   //   STRXui %xzr, %stack.0
3614   //
3615   // This also eliminates spilled cross register class COPYs (e.g. between x and
3616   // d regs) of the same size.  For example:
3617   //
3618   //   %0 = COPY %1; GPR64:%0, FPR64:%1
3619   //
3620   // will be filled as
3621   //
3622   //   LDRDui %0, fi<#0>
3623   //
3624   // instead of
3625   //
3626   //   LDRXui %Temp, fi<#0>
3627   //   %0 = FMOV %Temp
3628   //
3629   if (MI.isCopy() && Ops.size() == 1 &&
3630       // Make sure we're only folding the explicit COPY defs/uses.
3631       (Ops[0] == 0 || Ops[0] == 1)) {
3632     bool IsSpill = Ops[0] == 0;
3633     bool IsFill = !IsSpill;
3634     const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
3635     const MachineRegisterInfo &MRI = MF.getRegInfo();
3636     MachineBasicBlock &MBB = *MI.getParent();
3637     const MachineOperand &DstMO = MI.getOperand(0);
3638     const MachineOperand &SrcMO = MI.getOperand(1);
3639     Register DstReg = DstMO.getReg();
3640     Register SrcReg = SrcMO.getReg();
3641     // This is slightly expensive to compute for physical regs since
3642     // getMinimalPhysRegClass is slow.
3643     auto getRegClass = [&](unsigned Reg) {
3644       return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
3645                                               : TRI.getMinimalPhysRegClass(Reg);
3646     };
3647 
3648     if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
3649       assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
3650                  TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
3651              "Mismatched register size in non subreg COPY");
3652       if (IsSpill)
3653         storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
3654                             getRegClass(SrcReg), &TRI);
3655       else
3656         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
3657                              getRegClass(DstReg), &TRI);
3658       return &*--InsertPt;
3659     }
3660 
3661     // Handle cases like spilling def of:
3662     //
3663     //   %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
3664     //
3665     // where the physical register source can be widened and stored to the full
3666     // virtual reg destination stack slot, in this case producing:
3667     //
3668     //   STRXui %xzr, %stack.0
3669     //
3670     if (IsSpill && DstMO.isUndef() && Register::isPhysicalRegister(SrcReg)) {
3671       assert(SrcMO.getSubReg() == 0 &&
3672              "Unexpected subreg on physical register");
3673       const TargetRegisterClass *SpillRC;
3674       unsigned SpillSubreg;
3675       switch (DstMO.getSubReg()) {
3676       default:
3677         SpillRC = nullptr;
3678         break;
3679       case AArch64::sub_32:
3680       case AArch64::ssub:
3681         if (AArch64::GPR32RegClass.contains(SrcReg)) {
3682           SpillRC = &AArch64::GPR64RegClass;
3683           SpillSubreg = AArch64::sub_32;
3684         } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
3685           SpillRC = &AArch64::FPR64RegClass;
3686           SpillSubreg = AArch64::ssub;
3687         } else
3688           SpillRC = nullptr;
3689         break;
3690       case AArch64::dsub:
3691         if (AArch64::FPR64RegClass.contains(SrcReg)) {
3692           SpillRC = &AArch64::FPR128RegClass;
3693           SpillSubreg = AArch64::dsub;
3694         } else
3695           SpillRC = nullptr;
3696         break;
3697       }
3698 
3699       if (SpillRC)
3700         if (unsigned WidenedSrcReg =
3701                 TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
3702           storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
3703                               FrameIndex, SpillRC, &TRI);
3704           return &*--InsertPt;
3705         }
3706     }
3707 
3708     // Handle cases like filling use of:
3709     //
3710     //   %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
3711     //
3712     // where we can load the full virtual reg source stack slot, into the subreg
3713     // destination, in this case producing:
3714     //
3715     //   LDRWui %0:sub_32<def,read-undef>, %stack.0
3716     //
3717     if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
3718       const TargetRegisterClass *FillRC;
3719       switch (DstMO.getSubReg()) {
3720       default:
3721         FillRC = nullptr;
3722         break;
3723       case AArch64::sub_32:
3724         FillRC = &AArch64::GPR32RegClass;
3725         break;
3726       case AArch64::ssub:
3727         FillRC = &AArch64::FPR32RegClass;
3728         break;
3729       case AArch64::dsub:
3730         FillRC = &AArch64::FPR64RegClass;
3731         break;
3732       }
3733 
3734       if (FillRC) {
3735         assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
3736                    TRI.getRegSizeInBits(*FillRC) &&
3737                "Mismatched regclass size on folded subreg COPY");
3738         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC, &TRI);
3739         MachineInstr &LoadMI = *--InsertPt;
3740         MachineOperand &LoadDst = LoadMI.getOperand(0);
3741         assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
3742         LoadDst.setSubReg(DstMO.getSubReg());
3743         LoadDst.setIsUndef();
3744         return &LoadMI;
3745       }
3746     }
3747   }
3748 
3749   // Cannot fold.
3750   return nullptr;
3751 }
3752 
3753 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
3754                                     StackOffset &SOffset,
3755                                     bool *OutUseUnscaledOp,
3756                                     unsigned *OutUnscaledOp,
3757                                     int64_t *EmittableOffset) {
3758   // Set output values in case of early exit.
3759   if (EmittableOffset)
3760     *EmittableOffset = 0;
3761   if (OutUseUnscaledOp)
3762     *OutUseUnscaledOp = false;
3763   if (OutUnscaledOp)
3764     *OutUnscaledOp = 0;
3765 
3766   // Exit early for structured vector spills/fills as they can't take an
3767   // immediate offset.
3768   switch (MI.getOpcode()) {
3769   default:
3770     break;
3771   case AArch64::LD1Twov2d:
3772   case AArch64::LD1Threev2d:
3773   case AArch64::LD1Fourv2d:
3774   case AArch64::LD1Twov1d:
3775   case AArch64::LD1Threev1d:
3776   case AArch64::LD1Fourv1d:
3777   case AArch64::ST1Twov2d:
3778   case AArch64::ST1Threev2d:
3779   case AArch64::ST1Fourv2d:
3780   case AArch64::ST1Twov1d:
3781   case AArch64::ST1Threev1d:
3782   case AArch64::ST1Fourv1d:
3783   case AArch64::IRG:
3784   case AArch64::IRGstack:
3785   case AArch64::STGloop:
3786   case AArch64::STZGloop:
3787     return AArch64FrameOffsetCannotUpdate;
3788   }
3789 
3790   // Get the min/max offset and the scale.
3791   TypeSize ScaleValue(0U, false);
3792   unsigned Width;
3793   int64_t MinOff, MaxOff;
3794   if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
3795                                       MaxOff))
3796     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3797 
3798   // Construct the complete offset.
3799   bool IsMulVL = ScaleValue.isScalable();
3800   unsigned Scale = ScaleValue.getKnownMinSize();
3801   int64_t Offset = IsMulVL ? SOffset.getScalableBytes() : SOffset.getBytes();
3802 
3803   const MachineOperand &ImmOpnd =
3804       MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
3805   Offset += ImmOpnd.getImm() * Scale;
3806 
3807   // If the offset doesn't match the scale, we rewrite the instruction to
3808   // use the unscaled instruction instead. Likewise, if we have a negative
3809   // offset and there is an unscaled op to use.
3810   Optional<unsigned> UnscaledOp =
3811       AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
3812   bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
3813   if (useUnscaledOp &&
3814       !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
3815                                       MaxOff))
3816     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3817 
3818   Scale = ScaleValue.getKnownMinSize();
3819   assert(IsMulVL == ScaleValue.isScalable() &&
3820          "Unscaled opcode has different value for scalable");
3821 
3822   int64_t Remainder = Offset % Scale;
3823   assert(!(Remainder && useUnscaledOp) &&
3824          "Cannot have remainder when using unscaled op");
3825 
3826   assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
3827   int64_t NewOffset = Offset / Scale;
3828   if (MinOff <= NewOffset && NewOffset <= MaxOff)
3829     Offset = Remainder;
3830   else {
3831     NewOffset = NewOffset < 0 ? MinOff : MaxOff;
3832     Offset = Offset - NewOffset * Scale + Remainder;
3833   }
3834 
3835   if (EmittableOffset)
3836     *EmittableOffset = NewOffset;
3837   if (OutUseUnscaledOp)
3838     *OutUseUnscaledOp = useUnscaledOp;
3839   if (OutUnscaledOp && UnscaledOp)
3840     *OutUnscaledOp = *UnscaledOp;
3841 
3842   if (IsMulVL)
3843     SOffset = StackOffset(Offset, MVT::nxv1i8) +
3844               StackOffset(SOffset.getBytes(), MVT::i8);
3845   else
3846     SOffset = StackOffset(Offset, MVT::i8) +
3847               StackOffset(SOffset.getScalableBytes(), MVT::nxv1i8);
3848   return AArch64FrameOffsetCanUpdate |
3849          (SOffset ? 0 : AArch64FrameOffsetIsLegal);
3850 }
3851 
3852 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
3853                                     unsigned FrameReg, StackOffset &Offset,
3854                                     const AArch64InstrInfo *TII) {
3855   unsigned Opcode = MI.getOpcode();
3856   unsigned ImmIdx = FrameRegIdx + 1;
3857 
3858   if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
3859     Offset += StackOffset(MI.getOperand(ImmIdx).getImm(), MVT::i8);
3860     emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
3861                     MI.getOperand(0).getReg(), FrameReg, Offset, TII,
3862                     MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
3863     MI.eraseFromParent();
3864     Offset = StackOffset();
3865     return true;
3866   }
3867 
3868   int64_t NewOffset;
3869   unsigned UnscaledOp;
3870   bool UseUnscaledOp;
3871   int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
3872                                          &UnscaledOp, &NewOffset);
3873   if (Status & AArch64FrameOffsetCanUpdate) {
3874     if (Status & AArch64FrameOffsetIsLegal)
3875       // Replace the FrameIndex with FrameReg.
3876       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
3877     if (UseUnscaledOp)
3878       MI.setDesc(TII->get(UnscaledOp));
3879 
3880     MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
3881     return !Offset;
3882   }
3883 
3884   return false;
3885 }
3886 
3887 void AArch64InstrInfo::getNoop(MCInst &NopInst) const {
3888   NopInst.setOpcode(AArch64::HINT);
3889   NopInst.addOperand(MCOperand::createImm(0));
3890 }
3891 
3892 // AArch64 supports MachineCombiner.
3893 bool AArch64InstrInfo::useMachineCombiner() const { return true; }
3894 
3895 // True when Opc sets flag
3896 static bool isCombineInstrSettingFlag(unsigned Opc) {
3897   switch (Opc) {
3898   case AArch64::ADDSWrr:
3899   case AArch64::ADDSWri:
3900   case AArch64::ADDSXrr:
3901   case AArch64::ADDSXri:
3902   case AArch64::SUBSWrr:
3903   case AArch64::SUBSXrr:
3904   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3905   case AArch64::SUBSWri:
3906   case AArch64::SUBSXri:
3907     return true;
3908   default:
3909     break;
3910   }
3911   return false;
3912 }
3913 
3914 // 32b Opcodes that can be combined with a MUL
3915 static bool isCombineInstrCandidate32(unsigned Opc) {
3916   switch (Opc) {
3917   case AArch64::ADDWrr:
3918   case AArch64::ADDWri:
3919   case AArch64::SUBWrr:
3920   case AArch64::ADDSWrr:
3921   case AArch64::ADDSWri:
3922   case AArch64::SUBSWrr:
3923   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3924   case AArch64::SUBWri:
3925   case AArch64::SUBSWri:
3926     return true;
3927   default:
3928     break;
3929   }
3930   return false;
3931 }
3932 
3933 // 64b Opcodes that can be combined with a MUL
3934 static bool isCombineInstrCandidate64(unsigned Opc) {
3935   switch (Opc) {
3936   case AArch64::ADDXrr:
3937   case AArch64::ADDXri:
3938   case AArch64::SUBXrr:
3939   case AArch64::ADDSXrr:
3940   case AArch64::ADDSXri:
3941   case AArch64::SUBSXrr:
3942   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3943   case AArch64::SUBXri:
3944   case AArch64::SUBSXri:
3945   case AArch64::ADDv8i8:
3946   case AArch64::ADDv16i8:
3947   case AArch64::ADDv4i16:
3948   case AArch64::ADDv8i16:
3949   case AArch64::ADDv2i32:
3950   case AArch64::ADDv4i32:
3951   case AArch64::SUBv8i8:
3952   case AArch64::SUBv16i8:
3953   case AArch64::SUBv4i16:
3954   case AArch64::SUBv8i16:
3955   case AArch64::SUBv2i32:
3956   case AArch64::SUBv4i32:
3957     return true;
3958   default:
3959     break;
3960   }
3961   return false;
3962 }
3963 
3964 // FP Opcodes that can be combined with a FMUL.
3965 static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
3966   switch (Inst.getOpcode()) {
3967   default:
3968     break;
3969   case AArch64::FADDHrr:
3970   case AArch64::FADDSrr:
3971   case AArch64::FADDDrr:
3972   case AArch64::FADDv4f16:
3973   case AArch64::FADDv8f16:
3974   case AArch64::FADDv2f32:
3975   case AArch64::FADDv2f64:
3976   case AArch64::FADDv4f32:
3977   case AArch64::FSUBHrr:
3978   case AArch64::FSUBSrr:
3979   case AArch64::FSUBDrr:
3980   case AArch64::FSUBv4f16:
3981   case AArch64::FSUBv8f16:
3982   case AArch64::FSUBv2f32:
3983   case AArch64::FSUBv2f64:
3984   case AArch64::FSUBv4f32:
3985     TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
3986     // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
3987     // the target options or if FADD/FSUB has the contract fast-math flag.
3988     return Options.UnsafeFPMath ||
3989            Options.AllowFPOpFusion == FPOpFusion::Fast ||
3990            Inst.getFlag(MachineInstr::FmContract);
3991     return true;
3992   }
3993   return false;
3994 }
3995 
3996 // Opcodes that can be combined with a MUL
3997 static bool isCombineInstrCandidate(unsigned Opc) {
3998   return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
3999 }
4000 
4001 //
4002 // Utility routine that checks if \param MO is defined by an
4003 // \param CombineOpc instruction in the basic block \param MBB
4004 static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
4005                        unsigned CombineOpc, unsigned ZeroReg = 0,
4006                        bool CheckZeroReg = false) {
4007   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4008   MachineInstr *MI = nullptr;
4009 
4010   if (MO.isReg() && Register::isVirtualRegister(MO.getReg()))
4011     MI = MRI.getUniqueVRegDef(MO.getReg());
4012   // And it needs to be in the trace (otherwise, it won't have a depth).
4013   if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc)
4014     return false;
4015   // Must only used by the user we combine with.
4016   if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
4017     return false;
4018 
4019   if (CheckZeroReg) {
4020     assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
4021            MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
4022            MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
4023     // The third input reg must be zero.
4024     if (MI->getOperand(3).getReg() != ZeroReg)
4025       return false;
4026   }
4027 
4028   return true;
4029 }
4030 
4031 //
4032 // Is \param MO defined by an integer multiply and can be combined?
4033 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
4034                               unsigned MulOpc, unsigned ZeroReg) {
4035   return canCombine(MBB, MO, MulOpc, ZeroReg, true);
4036 }
4037 
4038 //
4039 // Is \param MO defined by a floating-point multiply and can be combined?
4040 static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
4041                                unsigned MulOpc) {
4042   return canCombine(MBB, MO, MulOpc);
4043 }
4044 
4045 // TODO: There are many more machine instruction opcodes to match:
4046 //       1. Other data types (integer, vectors)
4047 //       2. Other math / logic operations (xor, or)
4048 //       3. Other forms of the same operation (intrinsics and other variants)
4049 bool AArch64InstrInfo::isAssociativeAndCommutative(
4050     const MachineInstr &Inst) const {
4051   switch (Inst.getOpcode()) {
4052   case AArch64::FADDDrr:
4053   case AArch64::FADDSrr:
4054   case AArch64::FADDv2f32:
4055   case AArch64::FADDv2f64:
4056   case AArch64::FADDv4f32:
4057   case AArch64::FMULDrr:
4058   case AArch64::FMULSrr:
4059   case AArch64::FMULX32:
4060   case AArch64::FMULX64:
4061   case AArch64::FMULXv2f32:
4062   case AArch64::FMULXv2f64:
4063   case AArch64::FMULXv4f32:
4064   case AArch64::FMULv2f32:
4065   case AArch64::FMULv2f64:
4066   case AArch64::FMULv4f32:
4067     return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath;
4068   default:
4069     return false;
4070   }
4071 }
4072 
4073 /// Find instructions that can be turned into madd.
4074 static bool getMaddPatterns(MachineInstr &Root,
4075                             SmallVectorImpl<MachineCombinerPattern> &Patterns) {
4076   unsigned Opc = Root.getOpcode();
4077   MachineBasicBlock &MBB = *Root.getParent();
4078   bool Found = false;
4079 
4080   if (!isCombineInstrCandidate(Opc))
4081     return false;
4082   if (isCombineInstrSettingFlag(Opc)) {
4083     int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
4084     // When NZCV is live bail out.
4085     if (Cmp_NZCV == -1)
4086       return false;
4087     unsigned NewOpc = convertToNonFlagSettingOpc(Root);
4088     // When opcode can't change bail out.
4089     // CHECKME: do we miss any cases for opcode conversion?
4090     if (NewOpc == Opc)
4091       return false;
4092     Opc = NewOpc;
4093   }
4094 
4095   auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
4096                       MachineCombinerPattern Pattern) {
4097     if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
4098       Patterns.push_back(Pattern);
4099       Found = true;
4100     }
4101   };
4102 
4103   auto setVFound = [&](int Opcode, int Operand, MachineCombinerPattern Pattern) {
4104     if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
4105       Patterns.push_back(Pattern);
4106       Found = true;
4107     }
4108   };
4109 
4110   typedef MachineCombinerPattern MCP;
4111 
4112   switch (Opc) {
4113   default:
4114     break;
4115   case AArch64::ADDWrr:
4116     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4117            "ADDWrr does not have register operands");
4118     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
4119     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
4120     break;
4121   case AArch64::ADDXrr:
4122     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
4123     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
4124     break;
4125   case AArch64::SUBWrr:
4126     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
4127     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
4128     break;
4129   case AArch64::SUBXrr:
4130     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
4131     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
4132     break;
4133   case AArch64::ADDWri:
4134     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
4135     break;
4136   case AArch64::ADDXri:
4137     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
4138     break;
4139   case AArch64::SUBWri:
4140     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
4141     break;
4142   case AArch64::SUBXri:
4143     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
4144     break;
4145   case AArch64::ADDv8i8:
4146     setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
4147     setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
4148     break;
4149   case AArch64::ADDv16i8:
4150     setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
4151     setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
4152     break;
4153   case AArch64::ADDv4i16:
4154     setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
4155     setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
4156     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
4157     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
4158     break;
4159   case AArch64::ADDv8i16:
4160     setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
4161     setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
4162     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
4163     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
4164     break;
4165   case AArch64::ADDv2i32:
4166     setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
4167     setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
4168     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
4169     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
4170     break;
4171   case AArch64::ADDv4i32:
4172     setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
4173     setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
4174     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
4175     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
4176     break;
4177   case AArch64::SUBv8i8:
4178     setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
4179     setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
4180     break;
4181   case AArch64::SUBv16i8:
4182     setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
4183     setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
4184     break;
4185   case AArch64::SUBv4i16:
4186     setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
4187     setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
4188     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
4189     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
4190     break;
4191   case AArch64::SUBv8i16:
4192     setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
4193     setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
4194     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
4195     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
4196     break;
4197   case AArch64::SUBv2i32:
4198     setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
4199     setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
4200     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
4201     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
4202     break;
4203   case AArch64::SUBv4i32:
4204     setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
4205     setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
4206     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
4207     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
4208     break;
4209   }
4210   return Found;
4211 }
4212 /// Floating-Point Support
4213 
4214 /// Find instructions that can be turned into madd.
4215 static bool getFMAPatterns(MachineInstr &Root,
4216                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {
4217 
4218   if (!isCombineInstrCandidateFP(Root))
4219     return false;
4220 
4221   MachineBasicBlock &MBB = *Root.getParent();
4222   bool Found = false;
4223 
4224   auto Match = [&](int Opcode, int Operand,
4225                    MachineCombinerPattern Pattern) -> bool {
4226     if (canCombineWithFMUL(MBB, Root.getOperand(Operand), Opcode)) {
4227       Patterns.push_back(Pattern);
4228       return true;
4229     }
4230     return false;
4231   };
4232 
4233   typedef MachineCombinerPattern MCP;
4234 
4235   switch (Root.getOpcode()) {
4236   default:
4237     assert(false && "Unsupported FP instruction in combiner\n");
4238     break;
4239   case AArch64::FADDHrr:
4240     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4241            "FADDHrr does not have register operands");
4242 
4243     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
4244     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
4245     break;
4246   case AArch64::FADDSrr:
4247     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4248            "FADDSrr does not have register operands");
4249 
4250     Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
4251              Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
4252 
4253     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
4254              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
4255     break;
4256   case AArch64::FADDDrr:
4257     Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
4258              Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
4259 
4260     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
4261              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
4262     break;
4263   case AArch64::FADDv4f16:
4264     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
4265              Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
4266 
4267     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
4268              Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
4269     break;
4270   case AArch64::FADDv8f16:
4271     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
4272              Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
4273 
4274     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
4275              Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
4276     break;
4277   case AArch64::FADDv2f32:
4278     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
4279              Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
4280 
4281     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
4282              Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
4283     break;
4284   case AArch64::FADDv2f64:
4285     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
4286              Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
4287 
4288     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
4289              Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
4290     break;
4291   case AArch64::FADDv4f32:
4292     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
4293              Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
4294 
4295     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
4296              Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
4297     break;
4298   case AArch64::FSUBHrr:
4299     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
4300     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
4301     Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
4302     break;
4303   case AArch64::FSUBSrr:
4304     Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
4305 
4306     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
4307              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
4308 
4309     Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
4310     break;
4311   case AArch64::FSUBDrr:
4312     Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
4313 
4314     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
4315              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
4316 
4317     Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
4318     break;
4319   case AArch64::FSUBv4f16:
4320     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
4321              Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
4322 
4323     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
4324              Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
4325     break;
4326   case AArch64::FSUBv8f16:
4327     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
4328              Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
4329 
4330     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
4331              Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
4332     break;
4333   case AArch64::FSUBv2f32:
4334     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
4335              Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
4336 
4337     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
4338              Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
4339     break;
4340   case AArch64::FSUBv2f64:
4341     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
4342              Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
4343 
4344     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
4345              Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
4346     break;
4347   case AArch64::FSUBv4f32:
4348     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
4349              Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
4350 
4351     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
4352              Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
4353     break;
4354   }
4355   return Found;
4356 }
4357 
4358 /// Return true when a code sequence can improve throughput. It
4359 /// should be called only for instructions in loops.
4360 /// \param Pattern - combiner pattern
4361 bool AArch64InstrInfo::isThroughputPattern(
4362     MachineCombinerPattern Pattern) const {
4363   switch (Pattern) {
4364   default:
4365     break;
4366   case MachineCombinerPattern::FMULADDH_OP1:
4367   case MachineCombinerPattern::FMULADDH_OP2:
4368   case MachineCombinerPattern::FMULSUBH_OP1:
4369   case MachineCombinerPattern::FMULSUBH_OP2:
4370   case MachineCombinerPattern::FMULADDS_OP1:
4371   case MachineCombinerPattern::FMULADDS_OP2:
4372   case MachineCombinerPattern::FMULSUBS_OP1:
4373   case MachineCombinerPattern::FMULSUBS_OP2:
4374   case MachineCombinerPattern::FMULADDD_OP1:
4375   case MachineCombinerPattern::FMULADDD_OP2:
4376   case MachineCombinerPattern::FMULSUBD_OP1:
4377   case MachineCombinerPattern::FMULSUBD_OP2:
4378   case MachineCombinerPattern::FNMULSUBH_OP1:
4379   case MachineCombinerPattern::FNMULSUBS_OP1:
4380   case MachineCombinerPattern::FNMULSUBD_OP1:
4381   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
4382   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
4383   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
4384   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
4385   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4386   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4387   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
4388   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
4389   case MachineCombinerPattern::FMLAv4f16_OP2:
4390   case MachineCombinerPattern::FMLAv4f16_OP1:
4391   case MachineCombinerPattern::FMLAv8f16_OP1:
4392   case MachineCombinerPattern::FMLAv8f16_OP2:
4393   case MachineCombinerPattern::FMLAv2f32_OP2:
4394   case MachineCombinerPattern::FMLAv2f32_OP1:
4395   case MachineCombinerPattern::FMLAv2f64_OP1:
4396   case MachineCombinerPattern::FMLAv2f64_OP2:
4397   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
4398   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
4399   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
4400   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
4401   case MachineCombinerPattern::FMLAv4f32_OP1:
4402   case MachineCombinerPattern::FMLAv4f32_OP2:
4403   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
4404   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
4405   case MachineCombinerPattern::FMLSv4i16_indexed_OP1:
4406   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
4407   case MachineCombinerPattern::FMLSv8i16_indexed_OP1:
4408   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
4409   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
4410   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
4411   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
4412   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
4413   case MachineCombinerPattern::FMLSv4f16_OP1:
4414   case MachineCombinerPattern::FMLSv4f16_OP2:
4415   case MachineCombinerPattern::FMLSv8f16_OP1:
4416   case MachineCombinerPattern::FMLSv8f16_OP2:
4417   case MachineCombinerPattern::FMLSv2f32_OP2:
4418   case MachineCombinerPattern::FMLSv2f64_OP2:
4419   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
4420   case MachineCombinerPattern::FMLSv4f32_OP2:
4421   case MachineCombinerPattern::MULADDv8i8_OP1:
4422   case MachineCombinerPattern::MULADDv8i8_OP2:
4423   case MachineCombinerPattern::MULADDv16i8_OP1:
4424   case MachineCombinerPattern::MULADDv16i8_OP2:
4425   case MachineCombinerPattern::MULADDv4i16_OP1:
4426   case MachineCombinerPattern::MULADDv4i16_OP2:
4427   case MachineCombinerPattern::MULADDv8i16_OP1:
4428   case MachineCombinerPattern::MULADDv8i16_OP2:
4429   case MachineCombinerPattern::MULADDv2i32_OP1:
4430   case MachineCombinerPattern::MULADDv2i32_OP2:
4431   case MachineCombinerPattern::MULADDv4i32_OP1:
4432   case MachineCombinerPattern::MULADDv4i32_OP2:
4433   case MachineCombinerPattern::MULSUBv8i8_OP1:
4434   case MachineCombinerPattern::MULSUBv8i8_OP2:
4435   case MachineCombinerPattern::MULSUBv16i8_OP1:
4436   case MachineCombinerPattern::MULSUBv16i8_OP2:
4437   case MachineCombinerPattern::MULSUBv4i16_OP1:
4438   case MachineCombinerPattern::MULSUBv4i16_OP2:
4439   case MachineCombinerPattern::MULSUBv8i16_OP1:
4440   case MachineCombinerPattern::MULSUBv8i16_OP2:
4441   case MachineCombinerPattern::MULSUBv2i32_OP1:
4442   case MachineCombinerPattern::MULSUBv2i32_OP2:
4443   case MachineCombinerPattern::MULSUBv4i32_OP1:
4444   case MachineCombinerPattern::MULSUBv4i32_OP2:
4445   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4446   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4447   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4448   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4449   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4450   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4451   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4452   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4453   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4454   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4455   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4456   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4457   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4458   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4459   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4460   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4461     return true;
4462   } // end switch (Pattern)
4463   return false;
4464 }
4465 /// Return true when there is potentially a faster code sequence for an
4466 /// instruction chain ending in \p Root. All potential patterns are listed in
4467 /// the \p Pattern vector. Pattern should be sorted in priority order since the
4468 /// pattern evaluator stops checking as soon as it finds a faster sequence.
4469 
4470 bool AArch64InstrInfo::getMachineCombinerPatterns(
4471     MachineInstr &Root,
4472     SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
4473   // Integer patterns
4474   if (getMaddPatterns(Root, Patterns))
4475     return true;
4476   // Floating point patterns
4477   if (getFMAPatterns(Root, Patterns))
4478     return true;
4479 
4480   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
4481 }
4482 
4483 enum class FMAInstKind { Default, Indexed, Accumulator };
4484 /// genFusedMultiply - Generate fused multiply instructions.
4485 /// This function supports both integer and floating point instructions.
4486 /// A typical example:
4487 ///  F|MUL I=A,B,0
4488 ///  F|ADD R,I,C
4489 ///  ==> F|MADD R,A,B,C
4490 /// \param MF Containing MachineFunction
4491 /// \param MRI Register information
4492 /// \param TII Target information
4493 /// \param Root is the F|ADD instruction
4494 /// \param [out] InsInstrs is a vector of machine instructions and will
4495 /// contain the generated madd instruction
4496 /// \param IdxMulOpd is index of operand in Root that is the result of
4497 /// the F|MUL. In the example above IdxMulOpd is 1.
4498 /// \param MaddOpc the opcode fo the f|madd instruction
4499 /// \param RC Register class of operands
4500 /// \param kind of fma instruction (addressing mode) to be generated
4501 /// \param ReplacedAddend is the result register from the instruction
4502 /// replacing the non-combined operand, if any.
4503 static MachineInstr *
4504 genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
4505                  const TargetInstrInfo *TII, MachineInstr &Root,
4506                  SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
4507                  unsigned MaddOpc, const TargetRegisterClass *RC,
4508                  FMAInstKind kind = FMAInstKind::Default,
4509                  const Register *ReplacedAddend = nullptr) {
4510   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4511 
4512   unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
4513   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4514   Register ResultReg = Root.getOperand(0).getReg();
4515   Register SrcReg0 = MUL->getOperand(1).getReg();
4516   bool Src0IsKill = MUL->getOperand(1).isKill();
4517   Register SrcReg1 = MUL->getOperand(2).getReg();
4518   bool Src1IsKill = MUL->getOperand(2).isKill();
4519 
4520   unsigned SrcReg2;
4521   bool Src2IsKill;
4522   if (ReplacedAddend) {
4523     // If we just generated a new addend, we must be it's only use.
4524     SrcReg2 = *ReplacedAddend;
4525     Src2IsKill = true;
4526   } else {
4527     SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
4528     Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
4529   }
4530 
4531   if (Register::isVirtualRegister(ResultReg))
4532     MRI.constrainRegClass(ResultReg, RC);
4533   if (Register::isVirtualRegister(SrcReg0))
4534     MRI.constrainRegClass(SrcReg0, RC);
4535   if (Register::isVirtualRegister(SrcReg1))
4536     MRI.constrainRegClass(SrcReg1, RC);
4537   if (Register::isVirtualRegister(SrcReg2))
4538     MRI.constrainRegClass(SrcReg2, RC);
4539 
4540   MachineInstrBuilder MIB;
4541   if (kind == FMAInstKind::Default)
4542     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4543               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4544               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4545               .addReg(SrcReg2, getKillRegState(Src2IsKill));
4546   else if (kind == FMAInstKind::Indexed)
4547     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4548               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4549               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4550               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4551               .addImm(MUL->getOperand(3).getImm());
4552   else if (kind == FMAInstKind::Accumulator)
4553     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4554               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4555               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4556               .addReg(SrcReg1, getKillRegState(Src1IsKill));
4557   else
4558     assert(false && "Invalid FMA instruction kind \n");
4559   // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
4560   InsInstrs.push_back(MIB);
4561   return MUL;
4562 }
4563 
4564 /// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
4565 /// instructions.
4566 ///
4567 /// \see genFusedMultiply
4568 static MachineInstr *genFusedMultiplyAcc(
4569     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4570     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4571     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4572   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4573                           FMAInstKind::Accumulator);
4574 }
4575 
4576 /// genNeg - Helper to generate an intermediate negation of the second operand
4577 /// of Root
4578 static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
4579                        const TargetInstrInfo *TII, MachineInstr &Root,
4580                        SmallVectorImpl<MachineInstr *> &InsInstrs,
4581                        DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
4582                        unsigned MnegOpc, const TargetRegisterClass *RC) {
4583   Register NewVR = MRI.createVirtualRegister(RC);
4584   MachineInstrBuilder MIB =
4585       BuildMI(MF, Root.getDebugLoc(), TII->get(MnegOpc), NewVR)
4586           .add(Root.getOperand(2));
4587   InsInstrs.push_back(MIB);
4588 
4589   assert(InstrIdxForVirtReg.empty());
4590   InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4591 
4592   return NewVR;
4593 }
4594 
4595 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4596 /// instructions with an additional negation of the accumulator
4597 static MachineInstr *genFusedMultiplyAccNeg(
4598     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4599     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4600     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4601     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4602   assert(IdxMulOpd == 1);
4603 
4604   Register NewVR =
4605       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4606   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4607                           FMAInstKind::Accumulator, &NewVR);
4608 }
4609 
4610 /// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
4611 /// instructions.
4612 ///
4613 /// \see genFusedMultiply
4614 static MachineInstr *genFusedMultiplyIdx(
4615     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4616     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4617     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4618   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4619                           FMAInstKind::Indexed);
4620 }
4621 
4622 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4623 /// instructions with an additional negation of the accumulator
4624 static MachineInstr *genFusedMultiplyIdxNeg(
4625     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4626     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4627     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4628     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4629   assert(IdxMulOpd == 1);
4630 
4631   Register NewVR =
4632       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4633 
4634   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4635                           FMAInstKind::Indexed, &NewVR);
4636 }
4637 
4638 /// genMaddR - Generate madd instruction and combine mul and add using
4639 /// an extra virtual register
4640 /// Example - an ADD intermediate needs to be stored in a register:
4641 ///   MUL I=A,B,0
4642 ///   ADD R,I,Imm
4643 ///   ==> ORR  V, ZR, Imm
4644 ///   ==> MADD R,A,B,V
4645 /// \param MF Containing MachineFunction
4646 /// \param MRI Register information
4647 /// \param TII Target information
4648 /// \param Root is the ADD instruction
4649 /// \param [out] InsInstrs is a vector of machine instructions and will
4650 /// contain the generated madd instruction
4651 /// \param IdxMulOpd is index of operand in Root that is the result of
4652 /// the MUL. In the example above IdxMulOpd is 1.
4653 /// \param MaddOpc the opcode fo the madd instruction
4654 /// \param VR is a virtual register that holds the value of an ADD operand
4655 /// (V in the example above).
4656 /// \param RC Register class of operands
4657 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
4658                               const TargetInstrInfo *TII, MachineInstr &Root,
4659                               SmallVectorImpl<MachineInstr *> &InsInstrs,
4660                               unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
4661                               const TargetRegisterClass *RC) {
4662   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4663 
4664   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4665   Register ResultReg = Root.getOperand(0).getReg();
4666   Register SrcReg0 = MUL->getOperand(1).getReg();
4667   bool Src0IsKill = MUL->getOperand(1).isKill();
4668   Register SrcReg1 = MUL->getOperand(2).getReg();
4669   bool Src1IsKill = MUL->getOperand(2).isKill();
4670 
4671   if (Register::isVirtualRegister(ResultReg))
4672     MRI.constrainRegClass(ResultReg, RC);
4673   if (Register::isVirtualRegister(SrcReg0))
4674     MRI.constrainRegClass(SrcReg0, RC);
4675   if (Register::isVirtualRegister(SrcReg1))
4676     MRI.constrainRegClass(SrcReg1, RC);
4677   if (Register::isVirtualRegister(VR))
4678     MRI.constrainRegClass(VR, RC);
4679 
4680   MachineInstrBuilder MIB =
4681       BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4682           .addReg(SrcReg0, getKillRegState(Src0IsKill))
4683           .addReg(SrcReg1, getKillRegState(Src1IsKill))
4684           .addReg(VR);
4685   // Insert the MADD
4686   InsInstrs.push_back(MIB);
4687   return MUL;
4688 }
4689 
4690 /// When getMachineCombinerPatterns() finds potential patterns,
4691 /// this function generates the instructions that could replace the
4692 /// original code sequence
4693 void AArch64InstrInfo::genAlternativeCodeSequence(
4694     MachineInstr &Root, MachineCombinerPattern Pattern,
4695     SmallVectorImpl<MachineInstr *> &InsInstrs,
4696     SmallVectorImpl<MachineInstr *> &DelInstrs,
4697     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
4698   MachineBasicBlock &MBB = *Root.getParent();
4699   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4700   MachineFunction &MF = *MBB.getParent();
4701   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
4702 
4703   MachineInstr *MUL;
4704   const TargetRegisterClass *RC;
4705   unsigned Opc;
4706   switch (Pattern) {
4707   default:
4708     // Reassociate instructions.
4709     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
4710                                                 DelInstrs, InstrIdxForVirtReg);
4711     return;
4712   case MachineCombinerPattern::MULADDW_OP1:
4713   case MachineCombinerPattern::MULADDX_OP1:
4714     // MUL I=A,B,0
4715     // ADD R,I,C
4716     // ==> MADD R,A,B,C
4717     // --- Create(MADD);
4718     if (Pattern == MachineCombinerPattern::MULADDW_OP1) {
4719       Opc = AArch64::MADDWrrr;
4720       RC = &AArch64::GPR32RegClass;
4721     } else {
4722       Opc = AArch64::MADDXrrr;
4723       RC = &AArch64::GPR64RegClass;
4724     }
4725     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4726     break;
4727   case MachineCombinerPattern::MULADDW_OP2:
4728   case MachineCombinerPattern::MULADDX_OP2:
4729     // MUL I=A,B,0
4730     // ADD R,C,I
4731     // ==> MADD R,A,B,C
4732     // --- Create(MADD);
4733     if (Pattern == MachineCombinerPattern::MULADDW_OP2) {
4734       Opc = AArch64::MADDWrrr;
4735       RC = &AArch64::GPR32RegClass;
4736     } else {
4737       Opc = AArch64::MADDXrrr;
4738       RC = &AArch64::GPR64RegClass;
4739     }
4740     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4741     break;
4742   case MachineCombinerPattern::MULADDWI_OP1:
4743   case MachineCombinerPattern::MULADDXI_OP1: {
4744     // MUL I=A,B,0
4745     // ADD R,I,Imm
4746     // ==> ORR  V, ZR, Imm
4747     // ==> MADD R,A,B,V
4748     // --- Create(MADD);
4749     const TargetRegisterClass *OrrRC;
4750     unsigned BitSize, OrrOpc, ZeroReg;
4751     if (Pattern == MachineCombinerPattern::MULADDWI_OP1) {
4752       OrrOpc = AArch64::ORRWri;
4753       OrrRC = &AArch64::GPR32spRegClass;
4754       BitSize = 32;
4755       ZeroReg = AArch64::WZR;
4756       Opc = AArch64::MADDWrrr;
4757       RC = &AArch64::GPR32RegClass;
4758     } else {
4759       OrrOpc = AArch64::ORRXri;
4760       OrrRC = &AArch64::GPR64spRegClass;
4761       BitSize = 64;
4762       ZeroReg = AArch64::XZR;
4763       Opc = AArch64::MADDXrrr;
4764       RC = &AArch64::GPR64RegClass;
4765     }
4766     Register NewVR = MRI.createVirtualRegister(OrrRC);
4767     uint64_t Imm = Root.getOperand(2).getImm();
4768 
4769     if (Root.getOperand(3).isImm()) {
4770       unsigned Val = Root.getOperand(3).getImm();
4771       Imm = Imm << Val;
4772     }
4773     uint64_t UImm = SignExtend64(Imm, BitSize);
4774     uint64_t Encoding;
4775     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4776       MachineInstrBuilder MIB1 =
4777           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4778               .addReg(ZeroReg)
4779               .addImm(Encoding);
4780       InsInstrs.push_back(MIB1);
4781       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4782       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4783     }
4784     break;
4785   }
4786   case MachineCombinerPattern::MULSUBW_OP1:
4787   case MachineCombinerPattern::MULSUBX_OP1: {
4788     // MUL I=A,B,0
4789     // SUB R,I, C
4790     // ==> SUB  V, 0, C
4791     // ==> MADD R,A,B,V // = -C + A*B
4792     // --- Create(MADD);
4793     const TargetRegisterClass *SubRC;
4794     unsigned SubOpc, ZeroReg;
4795     if (Pattern == MachineCombinerPattern::MULSUBW_OP1) {
4796       SubOpc = AArch64::SUBWrr;
4797       SubRC = &AArch64::GPR32spRegClass;
4798       ZeroReg = AArch64::WZR;
4799       Opc = AArch64::MADDWrrr;
4800       RC = &AArch64::GPR32RegClass;
4801     } else {
4802       SubOpc = AArch64::SUBXrr;
4803       SubRC = &AArch64::GPR64spRegClass;
4804       ZeroReg = AArch64::XZR;
4805       Opc = AArch64::MADDXrrr;
4806       RC = &AArch64::GPR64RegClass;
4807     }
4808     Register NewVR = MRI.createVirtualRegister(SubRC);
4809     // SUB NewVR, 0, C
4810     MachineInstrBuilder MIB1 =
4811         BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
4812             .addReg(ZeroReg)
4813             .add(Root.getOperand(2));
4814     InsInstrs.push_back(MIB1);
4815     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4816     MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4817     break;
4818   }
4819   case MachineCombinerPattern::MULSUBW_OP2:
4820   case MachineCombinerPattern::MULSUBX_OP2:
4821     // MUL I=A,B,0
4822     // SUB R,C,I
4823     // ==> MSUB R,A,B,C (computes C - A*B)
4824     // --- Create(MSUB);
4825     if (Pattern == MachineCombinerPattern::MULSUBW_OP2) {
4826       Opc = AArch64::MSUBWrrr;
4827       RC = &AArch64::GPR32RegClass;
4828     } else {
4829       Opc = AArch64::MSUBXrrr;
4830       RC = &AArch64::GPR64RegClass;
4831     }
4832     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4833     break;
4834   case MachineCombinerPattern::MULSUBWI_OP1:
4835   case MachineCombinerPattern::MULSUBXI_OP1: {
4836     // MUL I=A,B,0
4837     // SUB R,I, Imm
4838     // ==> ORR  V, ZR, -Imm
4839     // ==> MADD R,A,B,V // = -Imm + A*B
4840     // --- Create(MADD);
4841     const TargetRegisterClass *OrrRC;
4842     unsigned BitSize, OrrOpc, ZeroReg;
4843     if (Pattern == MachineCombinerPattern::MULSUBWI_OP1) {
4844       OrrOpc = AArch64::ORRWri;
4845       OrrRC = &AArch64::GPR32spRegClass;
4846       BitSize = 32;
4847       ZeroReg = AArch64::WZR;
4848       Opc = AArch64::MADDWrrr;
4849       RC = &AArch64::GPR32RegClass;
4850     } else {
4851       OrrOpc = AArch64::ORRXri;
4852       OrrRC = &AArch64::GPR64spRegClass;
4853       BitSize = 64;
4854       ZeroReg = AArch64::XZR;
4855       Opc = AArch64::MADDXrrr;
4856       RC = &AArch64::GPR64RegClass;
4857     }
4858     Register NewVR = MRI.createVirtualRegister(OrrRC);
4859     uint64_t Imm = Root.getOperand(2).getImm();
4860     if (Root.getOperand(3).isImm()) {
4861       unsigned Val = Root.getOperand(3).getImm();
4862       Imm = Imm << Val;
4863     }
4864     uint64_t UImm = SignExtend64(-Imm, BitSize);
4865     uint64_t Encoding;
4866     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4867       MachineInstrBuilder MIB1 =
4868           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4869               .addReg(ZeroReg)
4870               .addImm(Encoding);
4871       InsInstrs.push_back(MIB1);
4872       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4873       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4874     }
4875     break;
4876   }
4877 
4878   case MachineCombinerPattern::MULADDv8i8_OP1:
4879     Opc = AArch64::MLAv8i8;
4880     RC = &AArch64::FPR64RegClass;
4881     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4882     break;
4883   case MachineCombinerPattern::MULADDv8i8_OP2:
4884     Opc = AArch64::MLAv8i8;
4885     RC = &AArch64::FPR64RegClass;
4886     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4887     break;
4888   case MachineCombinerPattern::MULADDv16i8_OP1:
4889     Opc = AArch64::MLAv16i8;
4890     RC = &AArch64::FPR128RegClass;
4891     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4892     break;
4893   case MachineCombinerPattern::MULADDv16i8_OP2:
4894     Opc = AArch64::MLAv16i8;
4895     RC = &AArch64::FPR128RegClass;
4896     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4897     break;
4898   case MachineCombinerPattern::MULADDv4i16_OP1:
4899     Opc = AArch64::MLAv4i16;
4900     RC = &AArch64::FPR64RegClass;
4901     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4902     break;
4903   case MachineCombinerPattern::MULADDv4i16_OP2:
4904     Opc = AArch64::MLAv4i16;
4905     RC = &AArch64::FPR64RegClass;
4906     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4907     break;
4908   case MachineCombinerPattern::MULADDv8i16_OP1:
4909     Opc = AArch64::MLAv8i16;
4910     RC = &AArch64::FPR128RegClass;
4911     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4912     break;
4913   case MachineCombinerPattern::MULADDv8i16_OP2:
4914     Opc = AArch64::MLAv8i16;
4915     RC = &AArch64::FPR128RegClass;
4916     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4917     break;
4918   case MachineCombinerPattern::MULADDv2i32_OP1:
4919     Opc = AArch64::MLAv2i32;
4920     RC = &AArch64::FPR64RegClass;
4921     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4922     break;
4923   case MachineCombinerPattern::MULADDv2i32_OP2:
4924     Opc = AArch64::MLAv2i32;
4925     RC = &AArch64::FPR64RegClass;
4926     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4927     break;
4928   case MachineCombinerPattern::MULADDv4i32_OP1:
4929     Opc = AArch64::MLAv4i32;
4930     RC = &AArch64::FPR128RegClass;
4931     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4932     break;
4933   case MachineCombinerPattern::MULADDv4i32_OP2:
4934     Opc = AArch64::MLAv4i32;
4935     RC = &AArch64::FPR128RegClass;
4936     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4937     break;
4938 
4939   case MachineCombinerPattern::MULSUBv8i8_OP1:
4940     Opc = AArch64::MLAv8i8;
4941     RC = &AArch64::FPR64RegClass;
4942     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4943                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
4944                                  RC);
4945     break;
4946   case MachineCombinerPattern::MULSUBv8i8_OP2:
4947     Opc = AArch64::MLSv8i8;
4948     RC = &AArch64::FPR64RegClass;
4949     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4950     break;
4951   case MachineCombinerPattern::MULSUBv16i8_OP1:
4952     Opc = AArch64::MLAv16i8;
4953     RC = &AArch64::FPR128RegClass;
4954     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4955                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
4956                                  RC);
4957     break;
4958   case MachineCombinerPattern::MULSUBv16i8_OP2:
4959     Opc = AArch64::MLSv16i8;
4960     RC = &AArch64::FPR128RegClass;
4961     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4962     break;
4963   case MachineCombinerPattern::MULSUBv4i16_OP1:
4964     Opc = AArch64::MLAv4i16;
4965     RC = &AArch64::FPR64RegClass;
4966     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4967                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4968                                  RC);
4969     break;
4970   case MachineCombinerPattern::MULSUBv4i16_OP2:
4971     Opc = AArch64::MLSv4i16;
4972     RC = &AArch64::FPR64RegClass;
4973     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4974     break;
4975   case MachineCombinerPattern::MULSUBv8i16_OP1:
4976     Opc = AArch64::MLAv8i16;
4977     RC = &AArch64::FPR128RegClass;
4978     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4979                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4980                                  RC);
4981     break;
4982   case MachineCombinerPattern::MULSUBv8i16_OP2:
4983     Opc = AArch64::MLSv8i16;
4984     RC = &AArch64::FPR128RegClass;
4985     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4986     break;
4987   case MachineCombinerPattern::MULSUBv2i32_OP1:
4988     Opc = AArch64::MLAv2i32;
4989     RC = &AArch64::FPR64RegClass;
4990     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4991                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4992                                  RC);
4993     break;
4994   case MachineCombinerPattern::MULSUBv2i32_OP2:
4995     Opc = AArch64::MLSv2i32;
4996     RC = &AArch64::FPR64RegClass;
4997     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4998     break;
4999   case MachineCombinerPattern::MULSUBv4i32_OP1:
5000     Opc = AArch64::MLAv4i32;
5001     RC = &AArch64::FPR128RegClass;
5002     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
5003                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
5004                                  RC);
5005     break;
5006   case MachineCombinerPattern::MULSUBv4i32_OP2:
5007     Opc = AArch64::MLSv4i32;
5008     RC = &AArch64::FPR128RegClass;
5009     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5010     break;
5011 
5012   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
5013     Opc = AArch64::MLAv4i16_indexed;
5014     RC = &AArch64::FPR64RegClass;
5015     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5016     break;
5017   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
5018     Opc = AArch64::MLAv4i16_indexed;
5019     RC = &AArch64::FPR64RegClass;
5020     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5021     break;
5022   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
5023     Opc = AArch64::MLAv8i16_indexed;
5024     RC = &AArch64::FPR128RegClass;
5025     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5026     break;
5027   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
5028     Opc = AArch64::MLAv8i16_indexed;
5029     RC = &AArch64::FPR128RegClass;
5030     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5031     break;
5032   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
5033     Opc = AArch64::MLAv2i32_indexed;
5034     RC = &AArch64::FPR64RegClass;
5035     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5036     break;
5037   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
5038     Opc = AArch64::MLAv2i32_indexed;
5039     RC = &AArch64::FPR64RegClass;
5040     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5041     break;
5042   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
5043     Opc = AArch64::MLAv4i32_indexed;
5044     RC = &AArch64::FPR128RegClass;
5045     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5046     break;
5047   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
5048     Opc = AArch64::MLAv4i32_indexed;
5049     RC = &AArch64::FPR128RegClass;
5050     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5051     break;
5052 
5053   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
5054     Opc = AArch64::MLAv4i16_indexed;
5055     RC = &AArch64::FPR64RegClass;
5056     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
5057                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
5058                                  RC);
5059     break;
5060   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
5061     Opc = AArch64::MLSv4i16_indexed;
5062     RC = &AArch64::FPR64RegClass;
5063     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5064     break;
5065   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
5066     Opc = AArch64::MLAv8i16_indexed;
5067     RC = &AArch64::FPR128RegClass;
5068     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
5069                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
5070                                  RC);
5071     break;
5072   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
5073     Opc = AArch64::MLSv8i16_indexed;
5074     RC = &AArch64::FPR128RegClass;
5075     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5076     break;
5077   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
5078     Opc = AArch64::MLAv2i32_indexed;
5079     RC = &AArch64::FPR64RegClass;
5080     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
5081                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
5082                                  RC);
5083     break;
5084   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
5085     Opc = AArch64::MLSv2i32_indexed;
5086     RC = &AArch64::FPR64RegClass;
5087     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5088     break;
5089   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
5090     Opc = AArch64::MLAv4i32_indexed;
5091     RC = &AArch64::FPR128RegClass;
5092     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
5093                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
5094                                  RC);
5095     break;
5096   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
5097     Opc = AArch64::MLSv4i32_indexed;
5098     RC = &AArch64::FPR128RegClass;
5099     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5100     break;
5101 
5102   // Floating Point Support
5103   case MachineCombinerPattern::FMULADDH_OP1:
5104     Opc = AArch64::FMADDHrrr;
5105     RC = &AArch64::FPR16RegClass;
5106     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5107     break;
5108   case MachineCombinerPattern::FMULADDS_OP1:
5109     Opc = AArch64::FMADDSrrr;
5110     RC = &AArch64::FPR32RegClass;
5111     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5112     break;
5113   case MachineCombinerPattern::FMULADDD_OP1:
5114     Opc = AArch64::FMADDDrrr;
5115     RC = &AArch64::FPR64RegClass;
5116     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5117     break;
5118 
5119   case MachineCombinerPattern::FMULADDH_OP2:
5120     Opc = AArch64::FMADDHrrr;
5121     RC = &AArch64::FPR16RegClass;
5122     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5123     break;
5124   case MachineCombinerPattern::FMULADDS_OP2:
5125     Opc = AArch64::FMADDSrrr;
5126     RC = &AArch64::FPR32RegClass;
5127     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5128     break;
5129   case MachineCombinerPattern::FMULADDD_OP2:
5130     Opc = AArch64::FMADDDrrr;
5131     RC = &AArch64::FPR64RegClass;
5132     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5133     break;
5134 
5135   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
5136     Opc = AArch64::FMLAv1i32_indexed;
5137     RC = &AArch64::FPR32RegClass;
5138     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5139                            FMAInstKind::Indexed);
5140     break;
5141   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
5142     Opc = AArch64::FMLAv1i32_indexed;
5143     RC = &AArch64::FPR32RegClass;
5144     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5145                            FMAInstKind::Indexed);
5146     break;
5147 
5148   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
5149     Opc = AArch64::FMLAv1i64_indexed;
5150     RC = &AArch64::FPR64RegClass;
5151     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5152                            FMAInstKind::Indexed);
5153     break;
5154   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
5155     Opc = AArch64::FMLAv1i64_indexed;
5156     RC = &AArch64::FPR64RegClass;
5157     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5158                            FMAInstKind::Indexed);
5159     break;
5160 
5161   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
5162     RC = &AArch64::FPR64RegClass;
5163     Opc = AArch64::FMLAv4i16_indexed;
5164     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5165                            FMAInstKind::Indexed);
5166     break;
5167   case MachineCombinerPattern::FMLAv4f16_OP1:
5168     RC = &AArch64::FPR64RegClass;
5169     Opc = AArch64::FMLAv4f16;
5170     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5171                            FMAInstKind::Accumulator);
5172     break;
5173   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
5174     RC = &AArch64::FPR64RegClass;
5175     Opc = AArch64::FMLAv4i16_indexed;
5176     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5177                            FMAInstKind::Indexed);
5178     break;
5179   case MachineCombinerPattern::FMLAv4f16_OP2:
5180     RC = &AArch64::FPR64RegClass;
5181     Opc = AArch64::FMLAv4f16;
5182     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5183                            FMAInstKind::Accumulator);
5184     break;
5185 
5186   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
5187   case MachineCombinerPattern::FMLAv2f32_OP1:
5188     RC = &AArch64::FPR64RegClass;
5189     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
5190       Opc = AArch64::FMLAv2i32_indexed;
5191       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5192                              FMAInstKind::Indexed);
5193     } else {
5194       Opc = AArch64::FMLAv2f32;
5195       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5196                              FMAInstKind::Accumulator);
5197     }
5198     break;
5199   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
5200   case MachineCombinerPattern::FMLAv2f32_OP2:
5201     RC = &AArch64::FPR64RegClass;
5202     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
5203       Opc = AArch64::FMLAv2i32_indexed;
5204       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5205                              FMAInstKind::Indexed);
5206     } else {
5207       Opc = AArch64::FMLAv2f32;
5208       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5209                              FMAInstKind::Accumulator);
5210     }
5211     break;
5212 
5213   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
5214     RC = &AArch64::FPR128RegClass;
5215     Opc = AArch64::FMLAv8i16_indexed;
5216     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5217                            FMAInstKind::Indexed);
5218     break;
5219   case MachineCombinerPattern::FMLAv8f16_OP1:
5220     RC = &AArch64::FPR128RegClass;
5221     Opc = AArch64::FMLAv8f16;
5222     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5223                            FMAInstKind::Accumulator);
5224     break;
5225   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
5226     RC = &AArch64::FPR128RegClass;
5227     Opc = AArch64::FMLAv8i16_indexed;
5228     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5229                            FMAInstKind::Indexed);
5230     break;
5231   case MachineCombinerPattern::FMLAv8f16_OP2:
5232     RC = &AArch64::FPR128RegClass;
5233     Opc = AArch64::FMLAv8f16;
5234     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5235                            FMAInstKind::Accumulator);
5236     break;
5237 
5238   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
5239   case MachineCombinerPattern::FMLAv2f64_OP1:
5240     RC = &AArch64::FPR128RegClass;
5241     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
5242       Opc = AArch64::FMLAv2i64_indexed;
5243       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5244                              FMAInstKind::Indexed);
5245     } else {
5246       Opc = AArch64::FMLAv2f64;
5247       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5248                              FMAInstKind::Accumulator);
5249     }
5250     break;
5251   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
5252   case MachineCombinerPattern::FMLAv2f64_OP2:
5253     RC = &AArch64::FPR128RegClass;
5254     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
5255       Opc = AArch64::FMLAv2i64_indexed;
5256       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5257                              FMAInstKind::Indexed);
5258     } else {
5259       Opc = AArch64::FMLAv2f64;
5260       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5261                              FMAInstKind::Accumulator);
5262     }
5263     break;
5264 
5265   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
5266   case MachineCombinerPattern::FMLAv4f32_OP1:
5267     RC = &AArch64::FPR128RegClass;
5268     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
5269       Opc = AArch64::FMLAv4i32_indexed;
5270       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5271                              FMAInstKind::Indexed);
5272     } else {
5273       Opc = AArch64::FMLAv4f32;
5274       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5275                              FMAInstKind::Accumulator);
5276     }
5277     break;
5278 
5279   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
5280   case MachineCombinerPattern::FMLAv4f32_OP2:
5281     RC = &AArch64::FPR128RegClass;
5282     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
5283       Opc = AArch64::FMLAv4i32_indexed;
5284       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5285                              FMAInstKind::Indexed);
5286     } else {
5287       Opc = AArch64::FMLAv4f32;
5288       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5289                              FMAInstKind::Accumulator);
5290     }
5291     break;
5292 
5293   case MachineCombinerPattern::FMULSUBH_OP1:
5294     Opc = AArch64::FNMSUBHrrr;
5295     RC = &AArch64::FPR16RegClass;
5296     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5297     break;
5298   case MachineCombinerPattern::FMULSUBS_OP1:
5299     Opc = AArch64::FNMSUBSrrr;
5300     RC = &AArch64::FPR32RegClass;
5301     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5302     break;
5303   case MachineCombinerPattern::FMULSUBD_OP1:
5304     Opc = AArch64::FNMSUBDrrr;
5305     RC = &AArch64::FPR64RegClass;
5306     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5307     break;
5308 
5309   case MachineCombinerPattern::FNMULSUBH_OP1:
5310     Opc = AArch64::FNMADDHrrr;
5311     RC = &AArch64::FPR16RegClass;
5312     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5313     break;
5314   case MachineCombinerPattern::FNMULSUBS_OP1:
5315     Opc = AArch64::FNMADDSrrr;
5316     RC = &AArch64::FPR32RegClass;
5317     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5318     break;
5319   case MachineCombinerPattern::FNMULSUBD_OP1:
5320     Opc = AArch64::FNMADDDrrr;
5321     RC = &AArch64::FPR64RegClass;
5322     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5323     break;
5324 
5325   case MachineCombinerPattern::FMULSUBH_OP2:
5326     Opc = AArch64::FMSUBHrrr;
5327     RC = &AArch64::FPR16RegClass;
5328     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5329     break;
5330   case MachineCombinerPattern::FMULSUBS_OP2:
5331     Opc = AArch64::FMSUBSrrr;
5332     RC = &AArch64::FPR32RegClass;
5333     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5334     break;
5335   case MachineCombinerPattern::FMULSUBD_OP2:
5336     Opc = AArch64::FMSUBDrrr;
5337     RC = &AArch64::FPR64RegClass;
5338     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5339     break;
5340 
5341   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
5342     Opc = AArch64::FMLSv1i32_indexed;
5343     RC = &AArch64::FPR32RegClass;
5344     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5345                            FMAInstKind::Indexed);
5346     break;
5347 
5348   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
5349     Opc = AArch64::FMLSv1i64_indexed;
5350     RC = &AArch64::FPR64RegClass;
5351     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5352                            FMAInstKind::Indexed);
5353     break;
5354 
5355   case MachineCombinerPattern::FMLSv4f16_OP1:
5356   case MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
5357     RC = &AArch64::FPR64RegClass;
5358     Register NewVR = MRI.createVirtualRegister(RC);
5359     MachineInstrBuilder MIB1 =
5360         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f16), NewVR)
5361             .add(Root.getOperand(2));
5362     InsInstrs.push_back(MIB1);
5363     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5364     if (Pattern == MachineCombinerPattern::FMLSv4f16_OP1) {
5365       Opc = AArch64::FMLAv4f16;
5366       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5367                              FMAInstKind::Accumulator, &NewVR);
5368     } else {
5369       Opc = AArch64::FMLAv4i16_indexed;
5370       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5371                              FMAInstKind::Indexed, &NewVR);
5372     }
5373     break;
5374   }
5375   case MachineCombinerPattern::FMLSv4f16_OP2:
5376     RC = &AArch64::FPR64RegClass;
5377     Opc = AArch64::FMLSv4f16;
5378     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5379                            FMAInstKind::Accumulator);
5380     break;
5381   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
5382     RC = &AArch64::FPR64RegClass;
5383     Opc = AArch64::FMLSv4i16_indexed;
5384     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5385                            FMAInstKind::Indexed);
5386     break;
5387 
5388   case MachineCombinerPattern::FMLSv2f32_OP2:
5389   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
5390     RC = &AArch64::FPR64RegClass;
5391     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
5392       Opc = AArch64::FMLSv2i32_indexed;
5393       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5394                              FMAInstKind::Indexed);
5395     } else {
5396       Opc = AArch64::FMLSv2f32;
5397       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5398                              FMAInstKind::Accumulator);
5399     }
5400     break;
5401 
5402   case MachineCombinerPattern::FMLSv8f16_OP1:
5403   case MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
5404     RC = &AArch64::FPR128RegClass;
5405     Register NewVR = MRI.createVirtualRegister(RC);
5406     MachineInstrBuilder MIB1 =
5407         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv8f16), NewVR)
5408             .add(Root.getOperand(2));
5409     InsInstrs.push_back(MIB1);
5410     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5411     if (Pattern == MachineCombinerPattern::FMLSv8f16_OP1) {
5412       Opc = AArch64::FMLAv8f16;
5413       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5414                              FMAInstKind::Accumulator, &NewVR);
5415     } else {
5416       Opc = AArch64::FMLAv8i16_indexed;
5417       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5418                              FMAInstKind::Indexed, &NewVR);
5419     }
5420     break;
5421   }
5422   case MachineCombinerPattern::FMLSv8f16_OP2:
5423     RC = &AArch64::FPR128RegClass;
5424     Opc = AArch64::FMLSv8f16;
5425     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5426                            FMAInstKind::Accumulator);
5427     break;
5428   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
5429     RC = &AArch64::FPR128RegClass;
5430     Opc = AArch64::FMLSv8i16_indexed;
5431     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5432                            FMAInstKind::Indexed);
5433     break;
5434 
5435   case MachineCombinerPattern::FMLSv2f64_OP2:
5436   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
5437     RC = &AArch64::FPR128RegClass;
5438     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
5439       Opc = AArch64::FMLSv2i64_indexed;
5440       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5441                              FMAInstKind::Indexed);
5442     } else {
5443       Opc = AArch64::FMLSv2f64;
5444       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5445                              FMAInstKind::Accumulator);
5446     }
5447     break;
5448 
5449   case MachineCombinerPattern::FMLSv4f32_OP2:
5450   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
5451     RC = &AArch64::FPR128RegClass;
5452     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
5453       Opc = AArch64::FMLSv4i32_indexed;
5454       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5455                              FMAInstKind::Indexed);
5456     } else {
5457       Opc = AArch64::FMLSv4f32;
5458       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5459                              FMAInstKind::Accumulator);
5460     }
5461     break;
5462   case MachineCombinerPattern::FMLSv2f32_OP1:
5463   case MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
5464     RC = &AArch64::FPR64RegClass;
5465     Register NewVR = MRI.createVirtualRegister(RC);
5466     MachineInstrBuilder MIB1 =
5467         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f32), NewVR)
5468             .add(Root.getOperand(2));
5469     InsInstrs.push_back(MIB1);
5470     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5471     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
5472       Opc = AArch64::FMLAv2i32_indexed;
5473       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5474                              FMAInstKind::Indexed, &NewVR);
5475     } else {
5476       Opc = AArch64::FMLAv2f32;
5477       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5478                              FMAInstKind::Accumulator, &NewVR);
5479     }
5480     break;
5481   }
5482   case MachineCombinerPattern::FMLSv4f32_OP1:
5483   case MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
5484     RC = &AArch64::FPR128RegClass;
5485     Register NewVR = MRI.createVirtualRegister(RC);
5486     MachineInstrBuilder MIB1 =
5487         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f32), NewVR)
5488             .add(Root.getOperand(2));
5489     InsInstrs.push_back(MIB1);
5490     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5491     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
5492       Opc = AArch64::FMLAv4i32_indexed;
5493       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5494                              FMAInstKind::Indexed, &NewVR);
5495     } else {
5496       Opc = AArch64::FMLAv4f32;
5497       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5498                              FMAInstKind::Accumulator, &NewVR);
5499     }
5500     break;
5501   }
5502   case MachineCombinerPattern::FMLSv2f64_OP1:
5503   case MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
5504     RC = &AArch64::FPR128RegClass;
5505     Register NewVR = MRI.createVirtualRegister(RC);
5506     MachineInstrBuilder MIB1 =
5507         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f64), NewVR)
5508             .add(Root.getOperand(2));
5509     InsInstrs.push_back(MIB1);
5510     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5511     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
5512       Opc = AArch64::FMLAv2i64_indexed;
5513       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5514                              FMAInstKind::Indexed, &NewVR);
5515     } else {
5516       Opc = AArch64::FMLAv2f64;
5517       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5518                              FMAInstKind::Accumulator, &NewVR);
5519     }
5520     break;
5521   }
5522   } // end switch (Pattern)
5523   // Record MUL and ADD/SUB for deletion
5524   DelInstrs.push_back(MUL);
5525   DelInstrs.push_back(&Root);
5526 }
5527 
5528 /// Replace csincr-branch sequence by simple conditional branch
5529 ///
5530 /// Examples:
5531 /// 1. \code
5532 ///   csinc  w9, wzr, wzr, <condition code>
5533 ///   tbnz   w9, #0, 0x44
5534 ///    \endcode
5535 /// to
5536 ///    \code
5537 ///   b.<inverted condition code>
5538 ///    \endcode
5539 ///
5540 /// 2. \code
5541 ///   csinc w9, wzr, wzr, <condition code>
5542 ///   tbz   w9, #0, 0x44
5543 ///    \endcode
5544 /// to
5545 ///    \code
5546 ///   b.<condition code>
5547 ///    \endcode
5548 ///
5549 /// Replace compare and branch sequence by TBZ/TBNZ instruction when the
5550 /// compare's constant operand is power of 2.
5551 ///
5552 /// Examples:
5553 ///    \code
5554 ///   and  w8, w8, #0x400
5555 ///   cbnz w8, L1
5556 ///    \endcode
5557 /// to
5558 ///    \code
5559 ///   tbnz w8, #10, L1
5560 ///    \endcode
5561 ///
5562 /// \param  MI Conditional Branch
5563 /// \return True when the simple conditional branch is generated
5564 ///
5565 bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
5566   bool IsNegativeBranch = false;
5567   bool IsTestAndBranch = false;
5568   unsigned TargetBBInMI = 0;
5569   switch (MI.getOpcode()) {
5570   default:
5571     llvm_unreachable("Unknown branch instruction?");
5572   case AArch64::Bcc:
5573     return false;
5574   case AArch64::CBZW:
5575   case AArch64::CBZX:
5576     TargetBBInMI = 1;
5577     break;
5578   case AArch64::CBNZW:
5579   case AArch64::CBNZX:
5580     TargetBBInMI = 1;
5581     IsNegativeBranch = true;
5582     break;
5583   case AArch64::TBZW:
5584   case AArch64::TBZX:
5585     TargetBBInMI = 2;
5586     IsTestAndBranch = true;
5587     break;
5588   case AArch64::TBNZW:
5589   case AArch64::TBNZX:
5590     TargetBBInMI = 2;
5591     IsNegativeBranch = true;
5592     IsTestAndBranch = true;
5593     break;
5594   }
5595   // So we increment a zero register and test for bits other
5596   // than bit 0? Conservatively bail out in case the verifier
5597   // missed this case.
5598   if (IsTestAndBranch && MI.getOperand(1).getImm())
5599     return false;
5600 
5601   // Find Definition.
5602   assert(MI.getParent() && "Incomplete machine instruciton\n");
5603   MachineBasicBlock *MBB = MI.getParent();
5604   MachineFunction *MF = MBB->getParent();
5605   MachineRegisterInfo *MRI = &MF->getRegInfo();
5606   Register VReg = MI.getOperand(0).getReg();
5607   if (!Register::isVirtualRegister(VReg))
5608     return false;
5609 
5610   MachineInstr *DefMI = MRI->getVRegDef(VReg);
5611 
5612   // Look through COPY instructions to find definition.
5613   while (DefMI->isCopy()) {
5614     Register CopyVReg = DefMI->getOperand(1).getReg();
5615     if (!MRI->hasOneNonDBGUse(CopyVReg))
5616       return false;
5617     if (!MRI->hasOneDef(CopyVReg))
5618       return false;
5619     DefMI = MRI->getVRegDef(CopyVReg);
5620   }
5621 
5622   switch (DefMI->getOpcode()) {
5623   default:
5624     return false;
5625   // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
5626   case AArch64::ANDWri:
5627   case AArch64::ANDXri: {
5628     if (IsTestAndBranch)
5629       return false;
5630     if (DefMI->getParent() != MBB)
5631       return false;
5632     if (!MRI->hasOneNonDBGUse(VReg))
5633       return false;
5634 
5635     bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
5636     uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
5637         DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
5638     if (!isPowerOf2_64(Mask))
5639       return false;
5640 
5641     MachineOperand &MO = DefMI->getOperand(1);
5642     Register NewReg = MO.getReg();
5643     if (!Register::isVirtualRegister(NewReg))
5644       return false;
5645 
5646     assert(!MRI->def_empty(NewReg) && "Register must be defined.");
5647 
5648     MachineBasicBlock &RefToMBB = *MBB;
5649     MachineBasicBlock *TBB = MI.getOperand(1).getMBB();
5650     DebugLoc DL = MI.getDebugLoc();
5651     unsigned Imm = Log2_64(Mask);
5652     unsigned Opc = (Imm < 32)
5653                        ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
5654                        : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
5655     MachineInstr *NewMI = BuildMI(RefToMBB, MI, DL, get(Opc))
5656                               .addReg(NewReg)
5657                               .addImm(Imm)
5658                               .addMBB(TBB);
5659     // Register lives on to the CBZ now.
5660     MO.setIsKill(false);
5661 
5662     // For immediate smaller than 32, we need to use the 32-bit
5663     // variant (W) in all cases. Indeed the 64-bit variant does not
5664     // allow to encode them.
5665     // Therefore, if the input register is 64-bit, we need to take the
5666     // 32-bit sub-part.
5667     if (!Is32Bit && Imm < 32)
5668       NewMI->getOperand(0).setSubReg(AArch64::sub_32);
5669     MI.eraseFromParent();
5670     return true;
5671   }
5672   // Look for CSINC
5673   case AArch64::CSINCWr:
5674   case AArch64::CSINCXr: {
5675     if (!(DefMI->getOperand(1).getReg() == AArch64::WZR &&
5676           DefMI->getOperand(2).getReg() == AArch64::WZR) &&
5677         !(DefMI->getOperand(1).getReg() == AArch64::XZR &&
5678           DefMI->getOperand(2).getReg() == AArch64::XZR))
5679       return false;
5680 
5681     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
5682       return false;
5683 
5684     AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(3).getImm();
5685     // Convert only when the condition code is not modified between
5686     // the CSINC and the branch. The CC may be used by other
5687     // instructions in between.
5688     if (areCFlagsAccessedBetweenInstrs(DefMI, MI, &getRegisterInfo(), AK_Write))
5689       return false;
5690     MachineBasicBlock &RefToMBB = *MBB;
5691     MachineBasicBlock *TBB = MI.getOperand(TargetBBInMI).getMBB();
5692     DebugLoc DL = MI.getDebugLoc();
5693     if (IsNegativeBranch)
5694       CC = AArch64CC::getInvertedCondCode(CC);
5695     BuildMI(RefToMBB, MI, DL, get(AArch64::Bcc)).addImm(CC).addMBB(TBB);
5696     MI.eraseFromParent();
5697     return true;
5698   }
5699   }
5700 }
5701 
5702 std::pair<unsigned, unsigned>
5703 AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5704   const unsigned Mask = AArch64II::MO_FRAGMENT;
5705   return std::make_pair(TF & Mask, TF & ~Mask);
5706 }
5707 
5708 ArrayRef<std::pair<unsigned, const char *>>
5709 AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5710   using namespace AArch64II;
5711 
5712   static const std::pair<unsigned, const char *> TargetFlags[] = {
5713       {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
5714       {MO_G3, "aarch64-g3"},     {MO_G2, "aarch64-g2"},
5715       {MO_G1, "aarch64-g1"},     {MO_G0, "aarch64-g0"},
5716       {MO_HI12, "aarch64-hi12"}};
5717   return makeArrayRef(TargetFlags);
5718 }
5719 
5720 ArrayRef<std::pair<unsigned, const char *>>
5721 AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5722   using namespace AArch64II;
5723 
5724   static const std::pair<unsigned, const char *> TargetFlags[] = {
5725       {MO_COFFSTUB, "aarch64-coffstub"},
5726       {MO_GOT, "aarch64-got"},
5727       {MO_NC, "aarch64-nc"},
5728       {MO_S, "aarch64-s"},
5729       {MO_TLS, "aarch64-tls"},
5730       {MO_DLLIMPORT, "aarch64-dllimport"},
5731       {MO_PREL, "aarch64-prel"},
5732       {MO_TAGGED, "aarch64-tagged"}};
5733   return makeArrayRef(TargetFlags);
5734 }
5735 
5736 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5737 AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5738   static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5739       {{MOSuppressPair, "aarch64-suppress-pair"},
5740        {MOStridedAccess, "aarch64-strided-access"}};
5741   return makeArrayRef(TargetFlags);
5742 }
5743 
5744 /// Constants defining how certain sequences should be outlined.
5745 /// This encompasses how an outlined function should be called, and what kind of
5746 /// frame should be emitted for that outlined function.
5747 ///
5748 /// \p MachineOutlinerDefault implies that the function should be called with
5749 /// a save and restore of LR to the stack.
5750 ///
5751 /// That is,
5752 ///
5753 /// I1     Save LR                    OUTLINED_FUNCTION:
5754 /// I2 --> BL OUTLINED_FUNCTION       I1
5755 /// I3     Restore LR                 I2
5756 ///                                   I3
5757 ///                                   RET
5758 ///
5759 /// * Call construction overhead: 3 (save + BL + restore)
5760 /// * Frame construction overhead: 1 (ret)
5761 /// * Requires stack fixups? Yes
5762 ///
5763 /// \p MachineOutlinerTailCall implies that the function is being created from
5764 /// a sequence of instructions ending in a return.
5765 ///
5766 /// That is,
5767 ///
5768 /// I1                             OUTLINED_FUNCTION:
5769 /// I2 --> B OUTLINED_FUNCTION     I1
5770 /// RET                            I2
5771 ///                                RET
5772 ///
5773 /// * Call construction overhead: 1 (B)
5774 /// * Frame construction overhead: 0 (Return included in sequence)
5775 /// * Requires stack fixups? No
5776 ///
5777 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5778 /// a BL instruction, but doesn't require LR to be saved and restored. This
5779 /// happens when LR is known to be dead.
5780 ///
5781 /// That is,
5782 ///
5783 /// I1                                OUTLINED_FUNCTION:
5784 /// I2 --> BL OUTLINED_FUNCTION       I1
5785 /// I3                                I2
5786 ///                                   I3
5787 ///                                   RET
5788 ///
5789 /// * Call construction overhead: 1 (BL)
5790 /// * Frame construction overhead: 1 (RET)
5791 /// * Requires stack fixups? No
5792 ///
5793 /// \p MachineOutlinerThunk implies that the function is being created from
5794 /// a sequence of instructions ending in a call. The outlined function is
5795 /// called with a BL instruction, and the outlined function tail-calls the
5796 /// original call destination.
5797 ///
5798 /// That is,
5799 ///
5800 /// I1                                OUTLINED_FUNCTION:
5801 /// I2 --> BL OUTLINED_FUNCTION       I1
5802 /// BL f                              I2
5803 ///                                   B f
5804 /// * Call construction overhead: 1 (BL)
5805 /// * Frame construction overhead: 0
5806 /// * Requires stack fixups? No
5807 ///
5808 /// \p MachineOutlinerRegSave implies that the function should be called with a
5809 /// save and restore of LR to an available register. This allows us to avoid
5810 /// stack fixups. Note that this outlining variant is compatible with the
5811 /// NoLRSave case.
5812 ///
5813 /// That is,
5814 ///
5815 /// I1     Save LR                    OUTLINED_FUNCTION:
5816 /// I2 --> BL OUTLINED_FUNCTION       I1
5817 /// I3     Restore LR                 I2
5818 ///                                   I3
5819 ///                                   RET
5820 ///
5821 /// * Call construction overhead: 3 (save + BL + restore)
5822 /// * Frame construction overhead: 1 (ret)
5823 /// * Requires stack fixups? No
5824 enum MachineOutlinerClass {
5825   MachineOutlinerDefault,  /// Emit a save, restore, call, and return.
5826   MachineOutlinerTailCall, /// Only emit a branch.
5827   MachineOutlinerNoLRSave, /// Emit a call and return.
5828   MachineOutlinerThunk,    /// Emit a call and tail-call.
5829   MachineOutlinerRegSave   /// Same as default, but save to a register.
5830 };
5831 
5832 enum MachineOutlinerMBBFlags {
5833   LRUnavailableSomewhere = 0x2,
5834   HasCalls = 0x4,
5835   UnsafeRegsDead = 0x8
5836 };
5837 
5838 unsigned
5839 AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5840   assert(C.LRUWasSet && "LRU wasn't set?");
5841   MachineFunction *MF = C.getMF();
5842   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
5843       MF->getSubtarget().getRegisterInfo());
5844 
5845   // Check if there is an available register across the sequence that we can
5846   // use.
5847   for (unsigned Reg : AArch64::GPR64RegClass) {
5848     if (!ARI->isReservedReg(*MF, Reg) &&
5849         Reg != AArch64::LR &&  // LR is not reserved, but don't use it.
5850         Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
5851         Reg != AArch64::X17 && // Ditto for X17.
5852         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5853       return Reg;
5854   }
5855 
5856   // No suitable register. Return 0.
5857   return 0u;
5858 }
5859 
5860 static bool
5861 outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
5862                                          const outliner::Candidate &b) {
5863   const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
5864   const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
5865 
5866   return MFIa->shouldSignReturnAddress(false) == MFIb->shouldSignReturnAddress(false) &&
5867          MFIa->shouldSignReturnAddress(true) == MFIb->shouldSignReturnAddress(true);
5868 }
5869 
5870 static bool
5871 outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
5872                                        const outliner::Candidate &b) {
5873   const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
5874   const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
5875 
5876   return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
5877 }
5878 
5879 static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
5880                                                 const outliner::Candidate &b) {
5881   const AArch64Subtarget &SubtargetA =
5882       a.getMF()->getSubtarget<AArch64Subtarget>();
5883   const AArch64Subtarget &SubtargetB =
5884       b.getMF()->getSubtarget<AArch64Subtarget>();
5885   return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
5886 }
5887 
5888 outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
5889     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5890   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5891   unsigned SequenceSize =
5892       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5893                       [this](unsigned Sum, const MachineInstr &MI) {
5894                         return Sum + getInstSizeInBytes(MI);
5895                       });
5896   unsigned NumBytesToCreateFrame = 0;
5897 
5898   // We only allow outlining for functions having exactly matching return
5899   // address signing attributes, i.e., all share the same value for the
5900   // attribute "sign-return-address" and all share the same type of key they
5901   // are signed with.
5902   // Additionally we require all functions to simultaniously either support
5903   // v8.3a features or not. Otherwise an outlined function could get signed
5904   // using dedicated v8.3 instructions and a call from a function that doesn't
5905   // support v8.3 instructions would therefore be invalid.
5906   if (std::adjacent_find(
5907           RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5908           [](const outliner::Candidate &a, const outliner::Candidate &b) {
5909             // Return true if a and b are non-equal w.r.t. return address
5910             // signing or support of v8.3a features
5911             if (outliningCandidatesSigningScopeConsensus(a, b) &&
5912                 outliningCandidatesSigningKeyConsensus(a, b) &&
5913                 outliningCandidatesV8_3OpsConsensus(a, b)) {
5914               return false;
5915             }
5916             return true;
5917           }) != RepeatedSequenceLocs.end()) {
5918     return outliner::OutlinedFunction();
5919   }
5920 
5921   // Since at this point all candidates agree on their return address signing
5922   // picking just one is fine. If the candidate functions potentially sign their
5923   // return addresses, the outlined function should do the same. Note that in
5924   // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5925   // not certainly true that the outlined function will have to sign its return
5926   // address but this decision is made later, when the decision to outline
5927   // has already been made.
5928   // The same holds for the number of additional instructions we need: On
5929   // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5930   // necessary. However, at this point we don't know if the outlined function
5931   // will have a RET instruction so we assume the worst.
5932   const TargetRegisterInfo &TRI = getRegisterInfo();
5933   if (FirstCand.getMF()
5934           ->getInfo<AArch64FunctionInfo>()
5935           ->shouldSignReturnAddress(true)) {
5936     // One PAC and one AUT instructions
5937     NumBytesToCreateFrame += 8;
5938 
5939     // We have to check if sp modifying instructions would get outlined.
5940     // If so we only allow outlining if sp is unchanged overall, so matching
5941     // sub and add instructions are okay to outline, all other sp modifications
5942     // are not
5943     auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5944       int SPValue = 0;
5945       MachineBasicBlock::iterator MBBI = C.front();
5946       for (;;) {
5947         if (MBBI->modifiesRegister(AArch64::SP, &TRI)) {
5948           switch (MBBI->getOpcode()) {
5949           case AArch64::ADDXri:
5950           case AArch64::ADDWri:
5951             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5952             assert(MBBI->getOperand(2).isImm() &&
5953                    "Expected operand to be immediate");
5954             assert(MBBI->getOperand(1).isReg() &&
5955                    "Expected operand to be a register");
5956             // Check if the add just increments sp. If so, we search for
5957             // matching sub instructions that decrement sp. If not, the
5958             // modification is illegal
5959             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5960               SPValue += MBBI->getOperand(2).getImm();
5961             else
5962               return true;
5963             break;
5964           case AArch64::SUBXri:
5965           case AArch64::SUBWri:
5966             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5967             assert(MBBI->getOperand(2).isImm() &&
5968                    "Expected operand to be immediate");
5969             assert(MBBI->getOperand(1).isReg() &&
5970                    "Expected operand to be a register");
5971             // Check if the sub just decrements sp. If so, we search for
5972             // matching add instructions that increment sp. If not, the
5973             // modification is illegal
5974             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5975               SPValue -= MBBI->getOperand(2).getImm();
5976             else
5977               return true;
5978             break;
5979           default:
5980             return true;
5981           }
5982         }
5983         if (MBBI == C.back())
5984           break;
5985         ++MBBI;
5986       }
5987       if (SPValue)
5988         return true;
5989       return false;
5990     };
5991     // Remove candidates with illegal stack modifying instructions
5992     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5993                                               RepeatedSequenceLocs.end(),
5994                                               hasIllegalSPModification),
5995                                RepeatedSequenceLocs.end());
5996 
5997     // If the sequence doesn't have enough candidates left, then we're done.
5998     if (RepeatedSequenceLocs.size() < 2)
5999       return outliner::OutlinedFunction();
6000   }
6001 
6002   // Properties about candidate MBBs that hold for all of them.
6003   unsigned FlagsSetInAll = 0xF;
6004 
6005   // Compute liveness information for each candidate, and set FlagsSetInAll.
6006   std::for_each(RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
6007                 [&FlagsSetInAll](outliner::Candidate &C) {
6008                   FlagsSetInAll &= C.Flags;
6009                 });
6010 
6011   // According to the AArch64 Procedure Call Standard, the following are
6012   // undefined on entry/exit from a function call:
6013   //
6014   // * Registers x16, x17, (and thus w16, w17)
6015   // * Condition codes (and thus the NZCV register)
6016   //
6017   // Because if this, we can't outline any sequence of instructions where
6018   // one
6019   // of these registers is live into/across it. Thus, we need to delete
6020   // those
6021   // candidates.
6022   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
6023     // If the unsafe registers in this block are all dead, then we don't need
6024     // to compute liveness here.
6025     if (C.Flags & UnsafeRegsDead)
6026       return false;
6027     C.initLRU(TRI);
6028     LiveRegUnits LRU = C.LRU;
6029     return (!LRU.available(AArch64::W16) || !LRU.available(AArch64::W17) ||
6030             !LRU.available(AArch64::NZCV));
6031   };
6032 
6033   // Are there any candidates where those registers are live?
6034   if (!(FlagsSetInAll & UnsafeRegsDead)) {
6035     // Erase every candidate that violates the restrictions above. (It could be
6036     // true that we have viable candidates, so it's not worth bailing out in
6037     // the case that, say, 1 out of 20 candidates violate the restructions.)
6038     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
6039                                               RepeatedSequenceLocs.end(),
6040                                               CantGuaranteeValueAcrossCall),
6041                                RepeatedSequenceLocs.end());
6042 
6043     // If the sequence doesn't have enough candidates left, then we're done.
6044     if (RepeatedSequenceLocs.size() < 2)
6045       return outliner::OutlinedFunction();
6046   }
6047 
6048   // At this point, we have only "safe" candidates to outline. Figure out
6049   // frame + call instruction information.
6050 
6051   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
6052 
6053   // Helper lambda which sets call information for every candidate.
6054   auto SetCandidateCallInfo =
6055       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
6056         for (outliner::Candidate &C : RepeatedSequenceLocs)
6057           C.setCallInfo(CallID, NumBytesForCall);
6058       };
6059 
6060   unsigned FrameID = MachineOutlinerDefault;
6061   NumBytesToCreateFrame += 4;
6062 
6063   bool HasBTI = any_of(RepeatedSequenceLocs, [](outliner::Candidate &C) {
6064     return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
6065   });
6066 
6067   // We check to see if CFI Instructions are present, and if they are
6068   // we find the number of CFI Instructions in the candidates.
6069   unsigned CFICount = 0;
6070   MachineBasicBlock::iterator MBBI = RepeatedSequenceLocs[0].front();
6071   for (unsigned Loc = RepeatedSequenceLocs[0].getStartIdx();
6072        Loc < RepeatedSequenceLocs[0].getEndIdx() + 1; Loc++) {
6073     const std::vector<MCCFIInstruction> &CFIInstructions =
6074         RepeatedSequenceLocs[0].getMF()->getFrameInstructions();
6075     if (MBBI->isCFIInstruction()) {
6076       unsigned CFIIndex = MBBI->getOperand(0).getCFIIndex();
6077       MCCFIInstruction CFI = CFIInstructions[CFIIndex];
6078       CFICount++;
6079     }
6080     MBBI++;
6081   }
6082 
6083   // We compare the number of found CFI Instructions to  the number of CFI
6084   // instructions in the parent function for each candidate.  We must check this
6085   // since if we outline one of the CFI instructions in a function, we have to
6086   // outline them all for correctness. If we do not, the address offsets will be
6087   // incorrect between the two sections of the program.
6088   for (outliner::Candidate &C : RepeatedSequenceLocs) {
6089     std::vector<MCCFIInstruction> CFIInstructions =
6090         C.getMF()->getFrameInstructions();
6091 
6092     if (CFICount > 0 && CFICount != CFIInstructions.size())
6093       return outliner::OutlinedFunction();
6094   }
6095 
6096   // Returns true if an instructions is safe to fix up, false otherwise.
6097   auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
6098     if (MI.isCall())
6099       return true;
6100 
6101     if (!MI.modifiesRegister(AArch64::SP, &TRI) &&
6102         !MI.readsRegister(AArch64::SP, &TRI))
6103       return true;
6104 
6105     // Any modification of SP will break our code to save/restore LR.
6106     // FIXME: We could handle some instructions which add a constant
6107     // offset to SP, with a bit more work.
6108     if (MI.modifiesRegister(AArch64::SP, &TRI))
6109       return false;
6110 
6111     // At this point, we have a stack instruction that we might need to
6112     // fix up. We'll handle it if it's a load or store.
6113     if (MI.mayLoadOrStore()) {
6114       const MachineOperand *Base; // Filled with the base operand of MI.
6115       int64_t Offset;             // Filled with the offset of MI.
6116       bool OffsetIsScalable;
6117 
6118       // Does it allow us to offset the base operand and is the base the
6119       // register SP?
6120       if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable, &TRI) ||
6121           !Base->isReg() || Base->getReg() != AArch64::SP)
6122         return false;
6123 
6124       // Fixe-up code below assumes bytes.
6125       if (OffsetIsScalable)
6126         return false;
6127 
6128       // Find the minimum/maximum offset for this instruction and check
6129       // if fixing it up would be in range.
6130       int64_t MinOffset,
6131           MaxOffset;  // Unscaled offsets for the instruction.
6132       TypeSize Scale(0U, false); // The scale to multiply the offsets by.
6133       unsigned DummyWidth;
6134       getMemOpInfo(MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
6135 
6136       Offset += 16; // Update the offset to what it would be if we outlined.
6137       if (Offset < MinOffset * (int64_t)Scale.getFixedSize() ||
6138           Offset > MaxOffset * (int64_t)Scale.getFixedSize())
6139         return false;
6140 
6141       // It's in range, so we can outline it.
6142       return true;
6143     }
6144 
6145     // FIXME: Add handling for instructions like "add x0, sp, #8".
6146 
6147     // We can't fix it up, so don't outline it.
6148     return false;
6149   };
6150 
6151   // True if it's possible to fix up each stack instruction in this sequence.
6152   // Important for frames/call variants that modify the stack.
6153   bool AllStackInstrsSafe = std::all_of(
6154       FirstCand.front(), std::next(FirstCand.back()), IsSafeToFixup);
6155 
6156   // If the last instruction in any candidate is a terminator, then we should
6157   // tail call all of the candidates.
6158   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
6159     FrameID = MachineOutlinerTailCall;
6160     NumBytesToCreateFrame = 0;
6161     SetCandidateCallInfo(MachineOutlinerTailCall, 4);
6162   }
6163 
6164   else if (LastInstrOpcode == AArch64::BL ||
6165            ((LastInstrOpcode == AArch64::BLR ||
6166              LastInstrOpcode == AArch64::BLRNoIP) &&
6167             !HasBTI)) {
6168     // FIXME: Do we need to check if the code after this uses the value of LR?
6169     FrameID = MachineOutlinerThunk;
6170     NumBytesToCreateFrame = 0;
6171     SetCandidateCallInfo(MachineOutlinerThunk, 4);
6172   }
6173 
6174   else {
6175     // We need to decide how to emit calls + frames. We can always emit the same
6176     // frame if we don't need to save to the stack. If we have to save to the
6177     // stack, then we need a different frame.
6178     unsigned NumBytesNoStackCalls = 0;
6179     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
6180 
6181     // Check if we have to save LR.
6182     for (outliner::Candidate &C : RepeatedSequenceLocs) {
6183       C.initLRU(TRI);
6184 
6185       // If we have a noreturn caller, then we're going to be conservative and
6186       // say that we have to save LR. If we don't have a ret at the end of the
6187       // block, then we can't reason about liveness accurately.
6188       //
6189       // FIXME: We can probably do better than always disabling this in
6190       // noreturn functions by fixing up the liveness info.
6191       bool IsNoReturn =
6192           C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
6193 
6194       // Is LR available? If so, we don't need a save.
6195       if (C.LRU.available(AArch64::LR) && !IsNoReturn) {
6196         NumBytesNoStackCalls += 4;
6197         C.setCallInfo(MachineOutlinerNoLRSave, 4);
6198         CandidatesWithoutStackFixups.push_back(C);
6199       }
6200 
6201       // Is an unused register available? If so, we won't modify the stack, so
6202       // we can outline with the same frame type as those that don't save LR.
6203       else if (findRegisterToSaveLRTo(C)) {
6204         NumBytesNoStackCalls += 12;
6205         C.setCallInfo(MachineOutlinerRegSave, 12);
6206         CandidatesWithoutStackFixups.push_back(C);
6207       }
6208 
6209       // Is SP used in the sequence at all? If not, we don't have to modify
6210       // the stack, so we are guaranteed to get the same frame.
6211       else if (C.UsedInSequence.available(AArch64::SP)) {
6212         NumBytesNoStackCalls += 12;
6213         C.setCallInfo(MachineOutlinerDefault, 12);
6214         CandidatesWithoutStackFixups.push_back(C);
6215       }
6216 
6217       // If we outline this, we need to modify the stack. Pretend we don't
6218       // outline this by saving all of its bytes.
6219       else {
6220         NumBytesNoStackCalls += SequenceSize;
6221       }
6222     }
6223 
6224     // If there are no places where we have to save LR, then note that we
6225     // don't have to update the stack. Otherwise, give every candidate the
6226     // default call type, as long as it's safe to do so.
6227     if (!AllStackInstrsSafe ||
6228         NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
6229       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
6230       FrameID = MachineOutlinerNoLRSave;
6231     } else {
6232       SetCandidateCallInfo(MachineOutlinerDefault, 12);
6233 
6234       // Bugzilla ID: 46767
6235       // TODO: Check if fixing up the stack more than once is safe so we can
6236       // outline these.
6237       //
6238       // An outline resulting in a caller that requires stack fixups at the
6239       // callsite to a callee that also requires stack fixups can happen when
6240       // there are no available registers at the candidate callsite for a
6241       // candidate that itself also has calls.
6242       //
6243       // In other words if function_containing_sequence in the following pseudo
6244       // assembly requires that we save LR at the point of the call, but there
6245       // are no available registers: in this case we save using SP and as a
6246       // result the SP offsets requires stack fixups by multiples of 16.
6247       //
6248       // function_containing_sequence:
6249       //   ...
6250       //   save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
6251       //   call OUTLINED_FUNCTION_N
6252       //   restore LR from SP
6253       //   ...
6254       //
6255       // OUTLINED_FUNCTION_N:
6256       //   save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
6257       //   ...
6258       //   bl foo
6259       //   restore LR from SP
6260       //   ret
6261       //
6262       // Because the code to handle more than one stack fixup does not
6263       // currently have the proper checks for legality, these cases will assert
6264       // in the AArch64 MachineOutliner. This is because the code to do this
6265       // needs more hardening, testing, better checks that generated code is
6266       // legal, etc and because it is only verified to handle a single pass of
6267       // stack fixup.
6268       //
6269       // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
6270       // these cases until they are known to be handled. Bugzilla 46767 is
6271       // referenced in comments at the assert site.
6272       //
6273       // To avoid asserting (or generating non-legal code on noassert builds)
6274       // we remove all candidates which would need more than one stack fixup by
6275       // pruning the cases where the candidate has calls while also having no
6276       // available LR and having no available general purpose registers to copy
6277       // LR to (ie one extra stack save/restore).
6278       //
6279       if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6280         erase_if(RepeatedSequenceLocs, [this](outliner::Candidate &C) {
6281           return (std::any_of(
6282                      C.front(), std::next(C.back()),
6283                      [](const MachineInstr &MI) { return MI.isCall(); })) &&
6284                  (!C.LRU.available(AArch64::LR) || !findRegisterToSaveLRTo(C));
6285         });
6286       }
6287     }
6288 
6289     // If we dropped all of the candidates, bail out here.
6290     if (RepeatedSequenceLocs.size() < 2) {
6291       RepeatedSequenceLocs.clear();
6292       return outliner::OutlinedFunction();
6293     }
6294   }
6295 
6296   // Does every candidate's MBB contain a call? If so, then we might have a call
6297   // in the range.
6298   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6299     // Check if the range contains a call. These require a save + restore of the
6300     // link register.
6301     bool ModStackToSaveLR = false;
6302     if (std::any_of(FirstCand.front(), FirstCand.back(),
6303                     [](const MachineInstr &MI) { return MI.isCall(); }))
6304       ModStackToSaveLR = true;
6305 
6306     // Handle the last instruction separately. If this is a tail call, then the
6307     // last instruction is a call. We don't want to save + restore in this case.
6308     // However, it could be possible that the last instruction is a call without
6309     // it being valid to tail call this sequence. We should consider this as
6310     // well.
6311     else if (FrameID != MachineOutlinerThunk &&
6312              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6313       ModStackToSaveLR = true;
6314 
6315     if (ModStackToSaveLR) {
6316       // We can't fix up the stack. Bail out.
6317       if (!AllStackInstrsSafe) {
6318         RepeatedSequenceLocs.clear();
6319         return outliner::OutlinedFunction();
6320       }
6321 
6322       // Save + restore LR.
6323       NumBytesToCreateFrame += 8;
6324     }
6325   }
6326 
6327   // If we have CFI instructions, we can only outline if the outlined section
6328   // can be a tail call
6329   if (FrameID != MachineOutlinerTailCall && CFICount > 0)
6330     return outliner::OutlinedFunction();
6331 
6332   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6333                                     NumBytesToCreateFrame, FrameID);
6334 }
6335 
6336 bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
6337     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6338   const Function &F = MF.getFunction();
6339 
6340   // Can F be deduplicated by the linker? If it can, don't outline from it.
6341   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6342     return false;
6343 
6344   // Don't outline from functions with section markings; the program could
6345   // expect that all the code is in the named section.
6346   // FIXME: Allow outlining from multiple functions with the same section
6347   // marking.
6348   if (F.hasSection())
6349     return false;
6350 
6351   // Outlining from functions with redzones is unsafe since the outliner may
6352   // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
6353   // outline from it.
6354   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
6355   if (!AFI || AFI->hasRedZone().getValueOr(true))
6356     return false;
6357 
6358   // FIXME: Teach the outliner to generate/handle Windows unwind info.
6359   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
6360     return false;
6361 
6362   // It's safe to outline from MF.
6363   return true;
6364 }
6365 
6366 bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6367                                               unsigned &Flags) const {
6368   // Check if LR is available through all of the MBB. If it's not, then set
6369   // a flag.
6370   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6371          "Suitable Machine Function for outlining must track liveness");
6372   LiveRegUnits LRU(getRegisterInfo());
6373 
6374   std::for_each(MBB.rbegin(), MBB.rend(),
6375                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6376 
6377   // Check if each of the unsafe registers are available...
6378   bool W16AvailableInBlock = LRU.available(AArch64::W16);
6379   bool W17AvailableInBlock = LRU.available(AArch64::W17);
6380   bool NZCVAvailableInBlock = LRU.available(AArch64::NZCV);
6381 
6382   // If all of these are dead (and not live out), we know we don't have to check
6383   // them later.
6384   if (W16AvailableInBlock && W17AvailableInBlock && NZCVAvailableInBlock)
6385     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6386 
6387   // Now, add the live outs to the set.
6388   LRU.addLiveOuts(MBB);
6389 
6390   // If any of these registers is available in the MBB, but also a live out of
6391   // the block, then we know outlining is unsafe.
6392   if (W16AvailableInBlock && !LRU.available(AArch64::W16))
6393     return false;
6394   if (W17AvailableInBlock && !LRU.available(AArch64::W17))
6395     return false;
6396   if (NZCVAvailableInBlock && !LRU.available(AArch64::NZCV))
6397     return false;
6398 
6399   // Check if there's a call inside this MachineBasicBlock. If there is, then
6400   // set a flag.
6401   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6402     Flags |= MachineOutlinerMBBFlags::HasCalls;
6403 
6404   MachineFunction *MF = MBB.getParent();
6405 
6406   // In the event that we outline, we may have to save LR. If there is an
6407   // available register in the MBB, then we'll always save LR there. Check if
6408   // this is true.
6409   bool CanSaveLR = false;
6410   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
6411       MF->getSubtarget().getRegisterInfo());
6412 
6413   // Check if there is an available register across the sequence that we can
6414   // use.
6415   for (unsigned Reg : AArch64::GPR64RegClass) {
6416     if (!ARI->isReservedReg(*MF, Reg) && Reg != AArch64::LR &&
6417         Reg != AArch64::X16 && Reg != AArch64::X17 && LRU.available(Reg)) {
6418       CanSaveLR = true;
6419       break;
6420     }
6421   }
6422 
6423   // Check if we have a register we can save LR to, and if LR was used
6424   // somewhere. If both of those things are true, then we need to evaluate the
6425   // safety of outlining stack instructions later.
6426   if (!CanSaveLR && !LRU.available(AArch64::LR))
6427     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6428 
6429   return true;
6430 }
6431 
6432 outliner::InstrType
6433 AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6434                                    unsigned Flags) const {
6435   MachineInstr &MI = *MIT;
6436   MachineBasicBlock *MBB = MI.getParent();
6437   MachineFunction *MF = MBB->getParent();
6438   AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
6439 
6440   // Don't outline anything used for return address signing. The outlined
6441   // function will get signed later if needed
6442   switch (MI.getOpcode()) {
6443   case AArch64::PACIASP:
6444   case AArch64::PACIBSP:
6445   case AArch64::AUTIASP:
6446   case AArch64::AUTIBSP:
6447   case AArch64::RETAA:
6448   case AArch64::RETAB:
6449   case AArch64::EMITBKEY:
6450     return outliner::InstrType::Illegal;
6451   }
6452 
6453   // Don't outline LOHs.
6454   if (FuncInfo->getLOHRelated().count(&MI))
6455     return outliner::InstrType::Illegal;
6456 
6457   // We can only outline these if we will tail call the outlined function, or
6458   // fix up the CFI offsets. Currently, CFI instructions are outlined only if
6459   // in a tail call.
6460   //
6461   // FIXME: If the proper fixups for the offset are implemented, this should be
6462   // possible.
6463   if (MI.isCFIInstruction())
6464     return outliner::InstrType::Legal;
6465 
6466   // Don't allow debug values to impact outlining type.
6467   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6468     return outliner::InstrType::Invisible;
6469 
6470   // At this point, KILL instructions don't really tell us much so we can go
6471   // ahead and skip over them.
6472   if (MI.isKill())
6473     return outliner::InstrType::Invisible;
6474 
6475   // Is this a terminator for a basic block?
6476   if (MI.isTerminator()) {
6477 
6478     // Is this the end of a function?
6479     if (MI.getParent()->succ_empty())
6480       return outliner::InstrType::Legal;
6481 
6482     // It's not, so don't outline it.
6483     return outliner::InstrType::Illegal;
6484   }
6485 
6486   // Make sure none of the operands are un-outlinable.
6487   for (const MachineOperand &MOP : MI.operands()) {
6488     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6489         MOP.isTargetIndex())
6490       return outliner::InstrType::Illegal;
6491 
6492     // If it uses LR or W30 explicitly, then don't touch it.
6493     if (MOP.isReg() && !MOP.isImplicit() &&
6494         (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
6495       return outliner::InstrType::Illegal;
6496   }
6497 
6498   // Special cases for instructions that can always be outlined, but will fail
6499   // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
6500   // be outlined because they don't require a *specific* value to be in LR.
6501   if (MI.getOpcode() == AArch64::ADRP)
6502     return outliner::InstrType::Legal;
6503 
6504   // If MI is a call we might be able to outline it. We don't want to outline
6505   // any calls that rely on the position of items on the stack. When we outline
6506   // something containing a call, we have to emit a save and restore of LR in
6507   // the outlined function. Currently, this always happens by saving LR to the
6508   // stack. Thus, if we outline, say, half the parameters for a function call
6509   // plus the call, then we'll break the callee's expectations for the layout
6510   // of the stack.
6511   //
6512   // FIXME: Allow calls to functions which construct a stack frame, as long
6513   // as they don't access arguments on the stack.
6514   // FIXME: Figure out some way to analyze functions defined in other modules.
6515   // We should be able to compute the memory usage based on the IR calling
6516   // convention, even if we can't see the definition.
6517   if (MI.isCall()) {
6518     // Get the function associated with the call. Look at each operand and find
6519     // the one that represents the callee and get its name.
6520     const Function *Callee = nullptr;
6521     for (const MachineOperand &MOP : MI.operands()) {
6522       if (MOP.isGlobal()) {
6523         Callee = dyn_cast<Function>(MOP.getGlobal());
6524         break;
6525       }
6526     }
6527 
6528     // Never outline calls to mcount.  There isn't any rule that would require
6529     // this, but the Linux kernel's "ftrace" feature depends on it.
6530     if (Callee && Callee->getName() == "\01_mcount")
6531       return outliner::InstrType::Illegal;
6532 
6533     // If we don't know anything about the callee, assume it depends on the
6534     // stack layout of the caller. In that case, it's only legal to outline
6535     // as a tail-call. Explicitly list the call instructions we know about so we
6536     // don't get unexpected results with call pseudo-instructions.
6537     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6538     if (MI.getOpcode() == AArch64::BLR ||
6539         MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
6540       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6541 
6542     if (!Callee)
6543       return UnknownCallOutlineType;
6544 
6545     // We have a function we have information about. Check it if it's something
6546     // can safely outline.
6547     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6548 
6549     // We don't know what's going on with the callee at all. Don't touch it.
6550     if (!CalleeMF)
6551       return UnknownCallOutlineType;
6552 
6553     // Check if we know anything about the callee saves on the function. If we
6554     // don't, then don't touch it, since that implies that we haven't
6555     // computed anything about its stack frame yet.
6556     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6557     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6558         MFI.getNumObjects() > 0)
6559       return UnknownCallOutlineType;
6560 
6561     // At this point, we can say that CalleeMF ought to not pass anything on the
6562     // stack. Therefore, we can outline it.
6563     return outliner::InstrType::Legal;
6564   }
6565 
6566   // Don't outline positions.
6567   if (MI.isPosition())
6568     return outliner::InstrType::Illegal;
6569 
6570   // Don't touch the link register or W30.
6571   if (MI.readsRegister(AArch64::W30, &getRegisterInfo()) ||
6572       MI.modifiesRegister(AArch64::W30, &getRegisterInfo()))
6573     return outliner::InstrType::Illegal;
6574 
6575   // Don't outline BTI instructions, because that will prevent the outlining
6576   // site from being indirectly callable.
6577   if (MI.getOpcode() == AArch64::HINT) {
6578     int64_t Imm = MI.getOperand(0).getImm();
6579     if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
6580       return outliner::InstrType::Illegal;
6581   }
6582 
6583   return outliner::InstrType::Legal;
6584 }
6585 
6586 void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6587   for (MachineInstr &MI : MBB) {
6588     const MachineOperand *Base;
6589     unsigned Width;
6590     int64_t Offset;
6591     bool OffsetIsScalable;
6592 
6593     // Is this a load or store with an immediate offset with SP as the base?
6594     if (!MI.mayLoadOrStore() ||
6595         !getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable, Width,
6596                                       &RI) ||
6597         (Base->isReg() && Base->getReg() != AArch64::SP))
6598       continue;
6599 
6600     // It is, so we have to fix it up.
6601     TypeSize Scale(0U, false);
6602     int64_t Dummy1, Dummy2;
6603 
6604     MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(MI);
6605     assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
6606     getMemOpInfo(MI.getOpcode(), Scale, Width, Dummy1, Dummy2);
6607     assert(Scale != 0 && "Unexpected opcode!");
6608     assert(!OffsetIsScalable && "Expected offset to be a byte offset");
6609 
6610     // We've pushed the return address to the stack, so add 16 to the offset.
6611     // This is safe, since we already checked if it would overflow when we
6612     // checked if this instruction was legal to outline.
6613     int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedSize();
6614     StackOffsetOperand.setImm(NewImm);
6615   }
6616 }
6617 
6618 static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
6619                                  bool ShouldSignReturnAddr,
6620                                  bool ShouldSignReturnAddrWithAKey) {
6621   if (ShouldSignReturnAddr) {
6622     MachineBasicBlock::iterator MBBPAC = MBB.begin();
6623     MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator();
6624     const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
6625     const TargetInstrInfo *TII = Subtarget.getInstrInfo();
6626     DebugLoc DL;
6627 
6628     if (MBBAUT != MBB.end())
6629       DL = MBBAUT->getDebugLoc();
6630 
6631     // At the very beginning of the basic block we insert the following
6632     // depending on the key type
6633     //
6634     // a_key:                   b_key:
6635     //    PACIASP                   EMITBKEY
6636     //    CFI_INSTRUCTION           PACIBSP
6637     //                              CFI_INSTRUCTION
6638     if (ShouldSignReturnAddrWithAKey) {
6639       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIASP))
6640           .setMIFlag(MachineInstr::FrameSetup);
6641     } else {
6642       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::EMITBKEY))
6643           .setMIFlag(MachineInstr::FrameSetup);
6644       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIBSP))
6645           .setMIFlag(MachineInstr::FrameSetup);
6646     }
6647     unsigned CFIIndex =
6648         MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
6649     BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::CFI_INSTRUCTION))
6650         .addCFIIndex(CFIIndex)
6651         .setMIFlags(MachineInstr::FrameSetup);
6652 
6653     // If v8.3a features are available we can replace a RET instruction by
6654     // RETAA or RETAB and omit the AUT instructions
6655     if (Subtarget.hasV8_3aOps() && MBBAUT != MBB.end() &&
6656         MBBAUT->getOpcode() == AArch64::RET) {
6657       BuildMI(MBB, MBBAUT, DL,
6658               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6659                                                     : AArch64::RETAB))
6660           .copyImplicitOps(*MBBAUT);
6661       MBB.erase(MBBAUT);
6662     } else {
6663       BuildMI(MBB, MBBAUT, DL,
6664               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6665                                                     : AArch64::AUTIBSP))
6666           .setMIFlag(MachineInstr::FrameDestroy);
6667     }
6668   }
6669 }
6670 
6671 void AArch64InstrInfo::buildOutlinedFrame(
6672     MachineBasicBlock &MBB, MachineFunction &MF,
6673     const outliner::OutlinedFunction &OF) const {
6674 
6675   AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
6676 
6677   if (OF.FrameConstructionID == MachineOutlinerTailCall)
6678     FI->setOutliningStyle("Tail Call");
6679   else if (OF.FrameConstructionID == MachineOutlinerThunk) {
6680     // For thunk outlining, rewrite the last instruction from a call to a
6681     // tail-call.
6682     MachineInstr *Call = &*--MBB.instr_end();
6683     unsigned TailOpcode;
6684     if (Call->getOpcode() == AArch64::BL) {
6685       TailOpcode = AArch64::TCRETURNdi;
6686     } else {
6687       assert(Call->getOpcode() == AArch64::BLR ||
6688              Call->getOpcode() == AArch64::BLRNoIP);
6689       TailOpcode = AArch64::TCRETURNriALL;
6690     }
6691     MachineInstr *TC = BuildMI(MF, DebugLoc(), get(TailOpcode))
6692                            .add(Call->getOperand(0))
6693                            .addImm(0);
6694     MBB.insert(MBB.end(), TC);
6695     Call->eraseFromParent();
6696 
6697     FI->setOutliningStyle("Thunk");
6698   }
6699 
6700   bool IsLeafFunction = true;
6701 
6702   // Is there a call in the outlined range?
6703   auto IsNonTailCall = [](const MachineInstr &MI) {
6704     return MI.isCall() && !MI.isReturn();
6705   };
6706 
6707   if (std::any_of(MBB.instr_begin(), MBB.instr_end(), IsNonTailCall)) {
6708     // Fix up the instructions in the range, since we're going to modify the
6709     // stack.
6710 
6711     // Bugzilla ID: 46767
6712     // TODO: Check if fixing up twice is safe so we can outline these.
6713     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6714            "Can only fix up stack references once");
6715     fixupPostOutline(MBB);
6716 
6717     IsLeafFunction = false;
6718 
6719     // LR has to be a live in so that we can save it.
6720     if (!MBB.isLiveIn(AArch64::LR))
6721       MBB.addLiveIn(AArch64::LR);
6722 
6723     MachineBasicBlock::iterator It = MBB.begin();
6724     MachineBasicBlock::iterator Et = MBB.end();
6725 
6726     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6727         OF.FrameConstructionID == MachineOutlinerThunk)
6728       Et = std::prev(MBB.end());
6729 
6730     // Insert a save before the outlined region
6731     MachineInstr *STRXpre = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6732                                 .addReg(AArch64::SP, RegState::Define)
6733                                 .addReg(AArch64::LR)
6734                                 .addReg(AArch64::SP)
6735                                 .addImm(-16);
6736     It = MBB.insert(It, STRXpre);
6737 
6738     const TargetSubtargetInfo &STI = MF.getSubtarget();
6739     const MCRegisterInfo *MRI = STI.getRegisterInfo();
6740     unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true);
6741 
6742     // Add a CFI saying the stack was moved 16 B down.
6743     int64_t StackPosEntry =
6744         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 16));
6745     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6746         .addCFIIndex(StackPosEntry)
6747         .setMIFlags(MachineInstr::FrameSetup);
6748 
6749     // Add a CFI saying that the LR that we want to find is now 16 B higher than
6750     // before.
6751     int64_t LRPosEntry =
6752         MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfReg, -16));
6753     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6754         .addCFIIndex(LRPosEntry)
6755         .setMIFlags(MachineInstr::FrameSetup);
6756 
6757     // Insert a restore before the terminator for the function.
6758     MachineInstr *LDRXpost = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6759                                  .addReg(AArch64::SP, RegState::Define)
6760                                  .addReg(AArch64::LR, RegState::Define)
6761                                  .addReg(AArch64::SP)
6762                                  .addImm(16);
6763     Et = MBB.insert(Et, LDRXpost);
6764   }
6765 
6766   // If a bunch of candidates reach this point they must agree on their return
6767   // address signing. It is therefore enough to just consider the signing
6768   // behaviour of one of them
6769   const auto &MFI = *OF.Candidates.front().getMF()->getInfo<AArch64FunctionInfo>();
6770   bool ShouldSignReturnAddr = MFI.shouldSignReturnAddress(!IsLeafFunction);
6771 
6772   // a_key is the default
6773   bool ShouldSignReturnAddrWithAKey = !MFI.shouldSignWithBKey();
6774 
6775   // If this is a tail call outlined function, then there's already a return.
6776   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6777       OF.FrameConstructionID == MachineOutlinerThunk) {
6778     signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6779                          ShouldSignReturnAddrWithAKey);
6780     return;
6781   }
6782 
6783   // It's not a tail call, so we have to insert the return ourselves.
6784 
6785   // LR has to be a live in so that we can return to it.
6786   if (!MBB.isLiveIn(AArch64::LR))
6787     MBB.addLiveIn(AArch64::LR);
6788 
6789   MachineInstr *ret = BuildMI(MF, DebugLoc(), get(AArch64::RET))
6790                           .addReg(AArch64::LR);
6791   MBB.insert(MBB.end(), ret);
6792 
6793   signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6794                        ShouldSignReturnAddrWithAKey);
6795 
6796   FI->setOutliningStyle("Function");
6797 
6798   // Did we have to modify the stack by saving the link register?
6799   if (OF.FrameConstructionID != MachineOutlinerDefault)
6800     return;
6801 
6802   // We modified the stack.
6803   // Walk over the basic block and fix up all the stack accesses.
6804   fixupPostOutline(MBB);
6805 }
6806 
6807 MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
6808     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6809     MachineFunction &MF, const outliner::Candidate &C) const {
6810 
6811   // Are we tail calling?
6812   if (C.CallConstructionID == MachineOutlinerTailCall) {
6813     // If yes, then we can just branch to the label.
6814     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::TCRETURNdi))
6815                             .addGlobalAddress(M.getNamedValue(MF.getName()))
6816                             .addImm(0));
6817     return It;
6818   }
6819 
6820   // Are we saving the link register?
6821   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6822       C.CallConstructionID == MachineOutlinerThunk) {
6823     // No, so just insert the call.
6824     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6825                             .addGlobalAddress(M.getNamedValue(MF.getName())));
6826     return It;
6827   }
6828 
6829   // We want to return the spot where we inserted the call.
6830   MachineBasicBlock::iterator CallPt;
6831 
6832   // Instructions for saving and restoring LR around the call instruction we're
6833   // going to insert.
6834   MachineInstr *Save;
6835   MachineInstr *Restore;
6836   // Can we save to a register?
6837   if (C.CallConstructionID == MachineOutlinerRegSave) {
6838     // FIXME: This logic should be sunk into a target-specific interface so that
6839     // we don't have to recompute the register.
6840     unsigned Reg = findRegisterToSaveLRTo(C);
6841     assert(Reg != 0 && "No callee-saved register available?");
6842 
6843     // Save and restore LR from that register.
6844     Save = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), Reg)
6845                .addReg(AArch64::XZR)
6846                .addReg(AArch64::LR)
6847                .addImm(0);
6848     Restore = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
6849                 .addReg(AArch64::XZR)
6850                 .addReg(Reg)
6851                 .addImm(0);
6852   } else {
6853     // We have the default case. Save and restore from SP.
6854     Save = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6855                .addReg(AArch64::SP, RegState::Define)
6856                .addReg(AArch64::LR)
6857                .addReg(AArch64::SP)
6858                .addImm(-16);
6859     Restore = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6860                   .addReg(AArch64::SP, RegState::Define)
6861                   .addReg(AArch64::LR, RegState::Define)
6862                   .addReg(AArch64::SP)
6863                   .addImm(16);
6864   }
6865 
6866   It = MBB.insert(It, Save);
6867   It++;
6868 
6869   // Insert the call.
6870   It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6871                           .addGlobalAddress(M.getNamedValue(MF.getName())));
6872   CallPt = It;
6873   It++;
6874 
6875   It = MBB.insert(It, Restore);
6876   return CallPt;
6877 }
6878 
6879 bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
6880   MachineFunction &MF) const {
6881   return MF.getFunction().hasMinSize();
6882 }
6883 
6884 Optional<DestSourcePair>
6885 AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
6886 
6887   // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
6888   // and zero immediate operands used as an alias for mov instruction.
6889   if (MI.getOpcode() == AArch64::ORRWrs &&
6890       MI.getOperand(1).getReg() == AArch64::WZR &&
6891       MI.getOperand(3).getImm() == 0x0) {
6892     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6893   }
6894 
6895   if (MI.getOpcode() == AArch64::ORRXrs &&
6896       MI.getOperand(1).getReg() == AArch64::XZR &&
6897       MI.getOperand(3).getImm() == 0x0) {
6898     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6899   }
6900 
6901   return None;
6902 }
6903 
6904 Optional<RegImmPair> AArch64InstrInfo::isAddImmediate(const MachineInstr &MI,
6905                                                       Register Reg) const {
6906   int Sign = 1;
6907   int64_t Offset = 0;
6908 
6909   // TODO: Handle cases where Reg is a super- or sub-register of the
6910   // destination register.
6911   const MachineOperand &Op0 = MI.getOperand(0);
6912   if (!Op0.isReg() || Reg != Op0.getReg())
6913     return None;
6914 
6915   switch (MI.getOpcode()) {
6916   default:
6917     return None;
6918   case AArch64::SUBWri:
6919   case AArch64::SUBXri:
6920   case AArch64::SUBSWri:
6921   case AArch64::SUBSXri:
6922     Sign *= -1;
6923     LLVM_FALLTHROUGH;
6924   case AArch64::ADDSWri:
6925   case AArch64::ADDSXri:
6926   case AArch64::ADDWri:
6927   case AArch64::ADDXri: {
6928     // TODO: Third operand can be global address (usually some string).
6929     if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg() ||
6930         !MI.getOperand(2).isImm())
6931       return None;
6932     int Shift = MI.getOperand(3).getImm();
6933     assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
6934     Offset = Sign * (MI.getOperand(2).getImm() << Shift);
6935   }
6936   }
6937   return RegImmPair{MI.getOperand(1).getReg(), Offset};
6938 }
6939 
6940 /// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
6941 /// the destination register then, if possible, describe the value in terms of
6942 /// the source register.
6943 static Optional<ParamLoadedValue>
6944 describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
6945                        const TargetInstrInfo *TII,
6946                        const TargetRegisterInfo *TRI) {
6947   auto DestSrc = TII->isCopyInstr(MI);
6948   if (!DestSrc)
6949     return None;
6950 
6951   Register DestReg = DestSrc->Destination->getReg();
6952   Register SrcReg = DestSrc->Source->getReg();
6953 
6954   auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
6955 
6956   // If the described register is the destination, just return the source.
6957   if (DestReg == DescribedReg)
6958     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6959 
6960   // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
6961   if (MI.getOpcode() == AArch64::ORRWrs &&
6962       TRI->isSuperRegister(DestReg, DescribedReg))
6963     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6964 
6965   // We may need to describe the lower part of a ORRXrs move.
6966   if (MI.getOpcode() == AArch64::ORRXrs &&
6967       TRI->isSubRegister(DestReg, DescribedReg)) {
6968     Register SrcSubReg = TRI->getSubReg(SrcReg, AArch64::sub_32);
6969     return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr);
6970   }
6971 
6972   assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
6973          "Unhandled ORR[XW]rs copy case");
6974 
6975   return None;
6976 }
6977 
6978 Optional<ParamLoadedValue>
6979 AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
6980                                       Register Reg) const {
6981   const MachineFunction *MF = MI.getMF();
6982   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6983   switch (MI.getOpcode()) {
6984   case AArch64::MOVZWi:
6985   case AArch64::MOVZXi: {
6986     // MOVZWi may be used for producing zero-extended 32-bit immediates in
6987     // 64-bit parameters, so we need to consider super-registers.
6988     if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
6989       return None;
6990 
6991     if (!MI.getOperand(1).isImm())
6992       return None;
6993     int64_t Immediate = MI.getOperand(1).getImm();
6994     int Shift = MI.getOperand(2).getImm();
6995     return ParamLoadedValue(MachineOperand::CreateImm(Immediate << Shift),
6996                             nullptr);
6997   }
6998   case AArch64::ORRWrs:
6999   case AArch64::ORRXrs:
7000     return describeORRLoadedValue(MI, Reg, this, TRI);
7001   }
7002 
7003   return TargetInstrInfo::describeLoadedValue(MI, Reg);
7004 }
7005 
7006 uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
7007   return get(Opc).TSFlags & AArch64::ElementSizeMask;
7008 }
7009 
7010 unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
7011   if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
7012     return AArch64::BLRNoIP;
7013   else
7014     return AArch64::BLR;
7015 }
7016 
7017 #define GET_INSTRINFO_HELPERS
7018 #define GET_INSTRMAP_INFO
7019 #include "AArch64GenInstrInfo.inc"
7020