1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64InstrInfo.h"
14 #include "AArch64MachineFunctionInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "Utils/AArch64BaseInfo.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineModuleInfo.h"
28 #include "llvm/CodeGen/MachineOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetRegisterInfo.h"
32 #include "llvm/CodeGen/TargetSubtargetInfo.h"
33 #include "llvm/IR/DebugInfoMetadata.h"
34 #include "llvm/IR/DebugLoc.h"
35 #include "llvm/IR/GlobalValue.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/MCInstrDesc.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Compiler.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <cassert>
48 #include <cstdint>
49 #include <iterator>
50 #include <utility>
51 
52 using namespace llvm;
53 
54 #define GET_INSTRINFO_CTOR_DTOR
55 #include "AArch64GenInstrInfo.inc"
56 
57 static cl::opt<unsigned> TBZDisplacementBits(
58     "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
59     cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
60 
61 static cl::opt<unsigned> CBZDisplacementBits(
62     "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
63     cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
64 
65 static cl::opt<unsigned>
66     BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
67                         cl::desc("Restrict range of Bcc instructions (DEBUG)"));
68 
69 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
70     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
71                           AArch64::CATCHRET),
72       RI(STI.getTargetTriple()), Subtarget(STI) {}
73 
74 /// GetInstSize - Return the number of bytes of code the specified
75 /// instruction may be.  This returns the maximum number of bytes.
76 unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
77   const MachineBasicBlock &MBB = *MI.getParent();
78   const MachineFunction *MF = MBB.getParent();
79   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
80 
81   {
82     auto Op = MI.getOpcode();
83     if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
84       return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
85   }
86 
87   // Meta-instructions emit no code.
88   if (MI.isMetaInstruction())
89     return 0;
90 
91   // FIXME: We currently only handle pseudoinstructions that don't get expanded
92   //        before the assembly printer.
93   unsigned NumBytes = 0;
94   const MCInstrDesc &Desc = MI.getDesc();
95   switch (Desc.getOpcode()) {
96   default:
97     // Anything not explicitly designated otherwise is a normal 4-byte insn.
98     NumBytes = 4;
99     break;
100   case TargetOpcode::STACKMAP:
101     // The upper bound for a stackmap intrinsic is the full length of its shadow
102     NumBytes = StackMapOpers(&MI).getNumPatchBytes();
103     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
104     break;
105   case TargetOpcode::PATCHPOINT:
106     // The size of the patchpoint intrinsic is the number of bytes requested
107     NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
108     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
109     break;
110   case TargetOpcode::STATEPOINT:
111     NumBytes = StatepointOpers(&MI).getNumPatchBytes();
112     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
113     // No patch bytes means a normal call inst is emitted
114     if (NumBytes == 0)
115       NumBytes = 4;
116     break;
117   case AArch64::TLSDESC_CALLSEQ:
118     // This gets lowered to an instruction sequence which takes 16 bytes
119     NumBytes = 16;
120     break;
121   case AArch64::SpeculationBarrierISBDSBEndBB:
122     // This gets lowered to 2 4-byte instructions.
123     NumBytes = 8;
124     break;
125   case AArch64::SpeculationBarrierSBEndBB:
126     // This gets lowered to 1 4-byte instructions.
127     NumBytes = 4;
128     break;
129   case AArch64::JumpTableDest32:
130   case AArch64::JumpTableDest16:
131   case AArch64::JumpTableDest8:
132     NumBytes = 12;
133     break;
134   case AArch64::SPACE:
135     NumBytes = MI.getOperand(1).getImm();
136     break;
137   case TargetOpcode::BUNDLE:
138     NumBytes = getInstBundleLength(MI);
139     break;
140   }
141 
142   return NumBytes;
143 }
144 
145 unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
146   unsigned Size = 0;
147   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
148   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
149   while (++I != E && I->isInsideBundle()) {
150     assert(!I->isBundle() && "No nested bundle!");
151     Size += getInstSizeInBytes(*I);
152   }
153   return Size;
154 }
155 
156 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
157                             SmallVectorImpl<MachineOperand> &Cond) {
158   // Block ends with fall-through condbranch.
159   switch (LastInst->getOpcode()) {
160   default:
161     llvm_unreachable("Unknown branch instruction?");
162   case AArch64::Bcc:
163     Target = LastInst->getOperand(1).getMBB();
164     Cond.push_back(LastInst->getOperand(0));
165     break;
166   case AArch64::CBZW:
167   case AArch64::CBZX:
168   case AArch64::CBNZW:
169   case AArch64::CBNZX:
170     Target = LastInst->getOperand(1).getMBB();
171     Cond.push_back(MachineOperand::CreateImm(-1));
172     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
173     Cond.push_back(LastInst->getOperand(0));
174     break;
175   case AArch64::TBZW:
176   case AArch64::TBZX:
177   case AArch64::TBNZW:
178   case AArch64::TBNZX:
179     Target = LastInst->getOperand(2).getMBB();
180     Cond.push_back(MachineOperand::CreateImm(-1));
181     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
182     Cond.push_back(LastInst->getOperand(0));
183     Cond.push_back(LastInst->getOperand(1));
184   }
185 }
186 
187 static unsigned getBranchDisplacementBits(unsigned Opc) {
188   switch (Opc) {
189   default:
190     llvm_unreachable("unexpected opcode!");
191   case AArch64::B:
192     return 64;
193   case AArch64::TBNZW:
194   case AArch64::TBZW:
195   case AArch64::TBNZX:
196   case AArch64::TBZX:
197     return TBZDisplacementBits;
198   case AArch64::CBNZW:
199   case AArch64::CBZW:
200   case AArch64::CBNZX:
201   case AArch64::CBZX:
202     return CBZDisplacementBits;
203   case AArch64::Bcc:
204     return BCCDisplacementBits;
205   }
206 }
207 
208 bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
209                                              int64_t BrOffset) const {
210   unsigned Bits = getBranchDisplacementBits(BranchOp);
211   assert(Bits >= 3 && "max branch displacement must be enough to jump"
212                       "over conditional branch expansion");
213   return isIntN(Bits, BrOffset / 4);
214 }
215 
216 MachineBasicBlock *
217 AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
218   switch (MI.getOpcode()) {
219   default:
220     llvm_unreachable("unexpected opcode!");
221   case AArch64::B:
222     return MI.getOperand(0).getMBB();
223   case AArch64::TBZW:
224   case AArch64::TBNZW:
225   case AArch64::TBZX:
226   case AArch64::TBNZX:
227     return MI.getOperand(2).getMBB();
228   case AArch64::CBZW:
229   case AArch64::CBNZW:
230   case AArch64::CBZX:
231   case AArch64::CBNZX:
232   case AArch64::Bcc:
233     return MI.getOperand(1).getMBB();
234   }
235 }
236 
237 // Branch analysis.
238 bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
239                                      MachineBasicBlock *&TBB,
240                                      MachineBasicBlock *&FBB,
241                                      SmallVectorImpl<MachineOperand> &Cond,
242                                      bool AllowModify) const {
243   // If the block has no terminators, it just falls into the block after it.
244   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
245   if (I == MBB.end())
246     return false;
247 
248   // Skip over SpeculationBarrierEndBB terminators
249   if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
250       I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
251     --I;
252   }
253 
254   if (!isUnpredicatedTerminator(*I))
255     return false;
256 
257   // Get the last instruction in the block.
258   MachineInstr *LastInst = &*I;
259 
260   // If there is only one terminator instruction, process it.
261   unsigned LastOpc = LastInst->getOpcode();
262   if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
263     if (isUncondBranchOpcode(LastOpc)) {
264       TBB = LastInst->getOperand(0).getMBB();
265       return false;
266     }
267     if (isCondBranchOpcode(LastOpc)) {
268       // Block ends with fall-through condbranch.
269       parseCondBranch(LastInst, TBB, Cond);
270       return false;
271     }
272     return true; // Can't handle indirect branch.
273   }
274 
275   // Get the instruction before it if it is a terminator.
276   MachineInstr *SecondLastInst = &*I;
277   unsigned SecondLastOpc = SecondLastInst->getOpcode();
278 
279   // If AllowModify is true and the block ends with two or more unconditional
280   // branches, delete all but the first unconditional branch.
281   if (AllowModify && isUncondBranchOpcode(LastOpc)) {
282     while (isUncondBranchOpcode(SecondLastOpc)) {
283       LastInst->eraseFromParent();
284       LastInst = SecondLastInst;
285       LastOpc = LastInst->getOpcode();
286       if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
287         // Return now the only terminator is an unconditional branch.
288         TBB = LastInst->getOperand(0).getMBB();
289         return false;
290       } else {
291         SecondLastInst = &*I;
292         SecondLastOpc = SecondLastInst->getOpcode();
293       }
294     }
295   }
296 
297   // If there are three terminators, we don't know what sort of block this is.
298   if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
299     return true;
300 
301   // If the block ends with a B and a Bcc, handle it.
302   if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
303     parseCondBranch(SecondLastInst, TBB, Cond);
304     FBB = LastInst->getOperand(0).getMBB();
305     return false;
306   }
307 
308   // If the block ends with two unconditional branches, handle it.  The second
309   // one is not executed, so remove it.
310   if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
311     TBB = SecondLastInst->getOperand(0).getMBB();
312     I = LastInst;
313     if (AllowModify)
314       I->eraseFromParent();
315     return false;
316   }
317 
318   // ...likewise if it ends with an indirect branch followed by an unconditional
319   // branch.
320   if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
321     I = LastInst;
322     if (AllowModify)
323       I->eraseFromParent();
324     return true;
325   }
326 
327   // Otherwise, can't handle this.
328   return true;
329 }
330 
331 bool AArch64InstrInfo::reverseBranchCondition(
332     SmallVectorImpl<MachineOperand> &Cond) const {
333   if (Cond[0].getImm() != -1) {
334     // Regular Bcc
335     AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
336     Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
337   } else {
338     // Folded compare-and-branch
339     switch (Cond[1].getImm()) {
340     default:
341       llvm_unreachable("Unknown conditional branch!");
342     case AArch64::CBZW:
343       Cond[1].setImm(AArch64::CBNZW);
344       break;
345     case AArch64::CBNZW:
346       Cond[1].setImm(AArch64::CBZW);
347       break;
348     case AArch64::CBZX:
349       Cond[1].setImm(AArch64::CBNZX);
350       break;
351     case AArch64::CBNZX:
352       Cond[1].setImm(AArch64::CBZX);
353       break;
354     case AArch64::TBZW:
355       Cond[1].setImm(AArch64::TBNZW);
356       break;
357     case AArch64::TBNZW:
358       Cond[1].setImm(AArch64::TBZW);
359       break;
360     case AArch64::TBZX:
361       Cond[1].setImm(AArch64::TBNZX);
362       break;
363     case AArch64::TBNZX:
364       Cond[1].setImm(AArch64::TBZX);
365       break;
366     }
367   }
368 
369   return false;
370 }
371 
372 unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
373                                         int *BytesRemoved) const {
374   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
375   if (I == MBB.end())
376     return 0;
377 
378   if (!isUncondBranchOpcode(I->getOpcode()) &&
379       !isCondBranchOpcode(I->getOpcode()))
380     return 0;
381 
382   // Remove the branch.
383   I->eraseFromParent();
384 
385   I = MBB.end();
386 
387   if (I == MBB.begin()) {
388     if (BytesRemoved)
389       *BytesRemoved = 4;
390     return 1;
391   }
392   --I;
393   if (!isCondBranchOpcode(I->getOpcode())) {
394     if (BytesRemoved)
395       *BytesRemoved = 4;
396     return 1;
397   }
398 
399   // Remove the branch.
400   I->eraseFromParent();
401   if (BytesRemoved)
402     *BytesRemoved = 8;
403 
404   return 2;
405 }
406 
407 void AArch64InstrInfo::instantiateCondBranch(
408     MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
409     ArrayRef<MachineOperand> Cond) const {
410   if (Cond[0].getImm() != -1) {
411     // Regular Bcc
412     BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
413   } else {
414     // Folded compare-and-branch
415     // Note that we use addOperand instead of addReg to keep the flags.
416     const MachineInstrBuilder MIB =
417         BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
418     if (Cond.size() > 3)
419       MIB.addImm(Cond[3].getImm());
420     MIB.addMBB(TBB);
421   }
422 }
423 
424 unsigned AArch64InstrInfo::insertBranch(
425     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
426     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
427   // Shouldn't be a fall through.
428   assert(TBB && "insertBranch must not be told to insert a fallthrough");
429 
430   if (!FBB) {
431     if (Cond.empty()) // Unconditional branch?
432       BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
433     else
434       instantiateCondBranch(MBB, DL, TBB, Cond);
435 
436     if (BytesAdded)
437       *BytesAdded = 4;
438 
439     return 1;
440   }
441 
442   // Two-way conditional branch.
443   instantiateCondBranch(MBB, DL, TBB, Cond);
444   BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
445 
446   if (BytesAdded)
447     *BytesAdded = 8;
448 
449   return 2;
450 }
451 
452 // Find the original register that VReg is copied from.
453 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
454   while (Register::isVirtualRegister(VReg)) {
455     const MachineInstr *DefMI = MRI.getVRegDef(VReg);
456     if (!DefMI->isFullCopy())
457       return VReg;
458     VReg = DefMI->getOperand(1).getReg();
459   }
460   return VReg;
461 }
462 
463 // Determine if VReg is defined by an instruction that can be folded into a
464 // csel instruction. If so, return the folded opcode, and the replacement
465 // register.
466 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
467                                 unsigned *NewVReg = nullptr) {
468   VReg = removeCopies(MRI, VReg);
469   if (!Register::isVirtualRegister(VReg))
470     return 0;
471 
472   bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
473   const MachineInstr *DefMI = MRI.getVRegDef(VReg);
474   unsigned Opc = 0;
475   unsigned SrcOpNum = 0;
476   switch (DefMI->getOpcode()) {
477   case AArch64::ADDSXri:
478   case AArch64::ADDSWri:
479     // if NZCV is used, do not fold.
480     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
481       return 0;
482     // fall-through to ADDXri and ADDWri.
483     LLVM_FALLTHROUGH;
484   case AArch64::ADDXri:
485   case AArch64::ADDWri:
486     // add x, 1 -> csinc.
487     if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
488         DefMI->getOperand(3).getImm() != 0)
489       return 0;
490     SrcOpNum = 1;
491     Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
492     break;
493 
494   case AArch64::ORNXrr:
495   case AArch64::ORNWrr: {
496     // not x -> csinv, represented as orn dst, xzr, src.
497     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
498     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
499       return 0;
500     SrcOpNum = 2;
501     Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
502     break;
503   }
504 
505   case AArch64::SUBSXrr:
506   case AArch64::SUBSWrr:
507     // if NZCV is used, do not fold.
508     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
509       return 0;
510     // fall-through to SUBXrr and SUBWrr.
511     LLVM_FALLTHROUGH;
512   case AArch64::SUBXrr:
513   case AArch64::SUBWrr: {
514     // neg x -> csneg, represented as sub dst, xzr, src.
515     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
516     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
517       return 0;
518     SrcOpNum = 2;
519     Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
520     break;
521   }
522   default:
523     return 0;
524   }
525   assert(Opc && SrcOpNum && "Missing parameters");
526 
527   if (NewVReg)
528     *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
529   return Opc;
530 }
531 
532 bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
533                                        ArrayRef<MachineOperand> Cond,
534                                        Register DstReg, Register TrueReg,
535                                        Register FalseReg, int &CondCycles,
536                                        int &TrueCycles,
537                                        int &FalseCycles) const {
538   // Check register classes.
539   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
540   const TargetRegisterClass *RC =
541       RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
542   if (!RC)
543     return false;
544 
545   // Also need to check the dest regclass, in case we're trying to optimize
546   // something like:
547   // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
548   if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
549     return false;
550 
551   // Expanding cbz/tbz requires an extra cycle of latency on the condition.
552   unsigned ExtraCondLat = Cond.size() != 1;
553 
554   // GPRs are handled by csel.
555   // FIXME: Fold in x+1, -x, and ~x when applicable.
556   if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
557       AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
558     // Single-cycle csel, csinc, csinv, and csneg.
559     CondCycles = 1 + ExtraCondLat;
560     TrueCycles = FalseCycles = 1;
561     if (canFoldIntoCSel(MRI, TrueReg))
562       TrueCycles = 0;
563     else if (canFoldIntoCSel(MRI, FalseReg))
564       FalseCycles = 0;
565     return true;
566   }
567 
568   // Scalar floating point is handled by fcsel.
569   // FIXME: Form fabs, fmin, and fmax when applicable.
570   if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
571       AArch64::FPR32RegClass.hasSubClassEq(RC)) {
572     CondCycles = 5 + ExtraCondLat;
573     TrueCycles = FalseCycles = 2;
574     return true;
575   }
576 
577   // Can't do vectors.
578   return false;
579 }
580 
581 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
582                                     MachineBasicBlock::iterator I,
583                                     const DebugLoc &DL, Register DstReg,
584                                     ArrayRef<MachineOperand> Cond,
585                                     Register TrueReg, Register FalseReg) const {
586   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
587 
588   // Parse the condition code, see parseCondBranch() above.
589   AArch64CC::CondCode CC;
590   switch (Cond.size()) {
591   default:
592     llvm_unreachable("Unknown condition opcode in Cond");
593   case 1: // b.cc
594     CC = AArch64CC::CondCode(Cond[0].getImm());
595     break;
596   case 3: { // cbz/cbnz
597     // We must insert a compare against 0.
598     bool Is64Bit;
599     switch (Cond[1].getImm()) {
600     default:
601       llvm_unreachable("Unknown branch opcode in Cond");
602     case AArch64::CBZW:
603       Is64Bit = false;
604       CC = AArch64CC::EQ;
605       break;
606     case AArch64::CBZX:
607       Is64Bit = true;
608       CC = AArch64CC::EQ;
609       break;
610     case AArch64::CBNZW:
611       Is64Bit = false;
612       CC = AArch64CC::NE;
613       break;
614     case AArch64::CBNZX:
615       Is64Bit = true;
616       CC = AArch64CC::NE;
617       break;
618     }
619     Register SrcReg = Cond[2].getReg();
620     if (Is64Bit) {
621       // cmp reg, #0 is actually subs xzr, reg, #0.
622       MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
623       BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
624           .addReg(SrcReg)
625           .addImm(0)
626           .addImm(0);
627     } else {
628       MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
629       BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
630           .addReg(SrcReg)
631           .addImm(0)
632           .addImm(0);
633     }
634     break;
635   }
636   case 4: { // tbz/tbnz
637     // We must insert a tst instruction.
638     switch (Cond[1].getImm()) {
639     default:
640       llvm_unreachable("Unknown branch opcode in Cond");
641     case AArch64::TBZW:
642     case AArch64::TBZX:
643       CC = AArch64CC::EQ;
644       break;
645     case AArch64::TBNZW:
646     case AArch64::TBNZX:
647       CC = AArch64CC::NE;
648       break;
649     }
650     // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
651     if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
652       BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
653           .addReg(Cond[2].getReg())
654           .addImm(
655               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
656     else
657       BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
658           .addReg(Cond[2].getReg())
659           .addImm(
660               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
661     break;
662   }
663   }
664 
665   unsigned Opc = 0;
666   const TargetRegisterClass *RC = nullptr;
667   bool TryFold = false;
668   if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
669     RC = &AArch64::GPR64RegClass;
670     Opc = AArch64::CSELXr;
671     TryFold = true;
672   } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
673     RC = &AArch64::GPR32RegClass;
674     Opc = AArch64::CSELWr;
675     TryFold = true;
676   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
677     RC = &AArch64::FPR64RegClass;
678     Opc = AArch64::FCSELDrrr;
679   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
680     RC = &AArch64::FPR32RegClass;
681     Opc = AArch64::FCSELSrrr;
682   }
683   assert(RC && "Unsupported regclass");
684 
685   // Try folding simple instructions into the csel.
686   if (TryFold) {
687     unsigned NewVReg = 0;
688     unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
689     if (FoldedOpc) {
690       // The folded opcodes csinc, csinc and csneg apply the operation to
691       // FalseReg, so we need to invert the condition.
692       CC = AArch64CC::getInvertedCondCode(CC);
693       TrueReg = FalseReg;
694     } else
695       FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
696 
697     // Fold the operation. Leave any dead instructions for DCE to clean up.
698     if (FoldedOpc) {
699       FalseReg = NewVReg;
700       Opc = FoldedOpc;
701       // The extends the live range of NewVReg.
702       MRI.clearKillFlags(NewVReg);
703     }
704   }
705 
706   // Pull all virtual register into the appropriate class.
707   MRI.constrainRegClass(TrueReg, RC);
708   MRI.constrainRegClass(FalseReg, RC);
709 
710   // Insert the csel.
711   BuildMI(MBB, I, DL, get(Opc), DstReg)
712       .addReg(TrueReg)
713       .addReg(FalseReg)
714       .addImm(CC);
715 }
716 
717 /// Returns true if a MOVi32imm or MOVi64imm can be expanded to an  ORRxx.
718 static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
719   uint64_t Imm = MI.getOperand(1).getImm();
720   uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
721   uint64_t Encoding;
722   return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
723 }
724 
725 // FIXME: this implementation should be micro-architecture dependent, so a
726 // micro-architecture target hook should be introduced here in future.
727 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
728   if (!Subtarget.hasCustomCheapAsMoveHandling())
729     return MI.isAsCheapAsAMove();
730 
731   const unsigned Opcode = MI.getOpcode();
732 
733   // Firstly, check cases gated by features.
734 
735   if (Subtarget.hasZeroCycleZeroingFP()) {
736     if (Opcode == AArch64::FMOVH0 ||
737         Opcode == AArch64::FMOVS0 ||
738         Opcode == AArch64::FMOVD0)
739       return true;
740   }
741 
742   if (Subtarget.hasZeroCycleZeroingGP()) {
743     if (Opcode == TargetOpcode::COPY &&
744         (MI.getOperand(1).getReg() == AArch64::WZR ||
745          MI.getOperand(1).getReg() == AArch64::XZR))
746       return true;
747   }
748 
749   // Secondly, check cases specific to sub-targets.
750 
751   if (Subtarget.hasExynosCheapAsMoveHandling()) {
752     if (isExynosCheapAsMove(MI))
753       return true;
754 
755     return MI.isAsCheapAsAMove();
756   }
757 
758   // Finally, check generic cases.
759 
760   switch (Opcode) {
761   default:
762     return false;
763 
764   // add/sub on register without shift
765   case AArch64::ADDWri:
766   case AArch64::ADDXri:
767   case AArch64::SUBWri:
768   case AArch64::SUBXri:
769     return (MI.getOperand(3).getImm() == 0);
770 
771   // logical ops on immediate
772   case AArch64::ANDWri:
773   case AArch64::ANDXri:
774   case AArch64::EORWri:
775   case AArch64::EORXri:
776   case AArch64::ORRWri:
777   case AArch64::ORRXri:
778     return true;
779 
780   // logical ops on register without shift
781   case AArch64::ANDWrr:
782   case AArch64::ANDXrr:
783   case AArch64::BICWrr:
784   case AArch64::BICXrr:
785   case AArch64::EONWrr:
786   case AArch64::EONXrr:
787   case AArch64::EORWrr:
788   case AArch64::EORXrr:
789   case AArch64::ORNWrr:
790   case AArch64::ORNXrr:
791   case AArch64::ORRWrr:
792   case AArch64::ORRXrr:
793     return true;
794 
795   // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
796   // ORRXri, it is as cheap as MOV
797   case AArch64::MOVi32imm:
798     return canBeExpandedToORR(MI, 32);
799   case AArch64::MOVi64imm:
800     return canBeExpandedToORR(MI, 64);
801   }
802 
803   llvm_unreachable("Unknown opcode to check as cheap as a move!");
804 }
805 
806 bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
807   switch (MI.getOpcode()) {
808   default:
809     return false;
810 
811   case AArch64::ADDWrs:
812   case AArch64::ADDXrs:
813   case AArch64::ADDSWrs:
814   case AArch64::ADDSXrs: {
815     unsigned Imm = MI.getOperand(3).getImm();
816     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
817     if (ShiftVal == 0)
818       return true;
819     return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
820   }
821 
822   case AArch64::ADDWrx:
823   case AArch64::ADDXrx:
824   case AArch64::ADDXrx64:
825   case AArch64::ADDSWrx:
826   case AArch64::ADDSXrx:
827   case AArch64::ADDSXrx64: {
828     unsigned Imm = MI.getOperand(3).getImm();
829     switch (AArch64_AM::getArithExtendType(Imm)) {
830     default:
831       return false;
832     case AArch64_AM::UXTB:
833     case AArch64_AM::UXTH:
834     case AArch64_AM::UXTW:
835     case AArch64_AM::UXTX:
836       return AArch64_AM::getArithShiftValue(Imm) <= 4;
837     }
838   }
839 
840   case AArch64::SUBWrs:
841   case AArch64::SUBSWrs: {
842     unsigned Imm = MI.getOperand(3).getImm();
843     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
844     return ShiftVal == 0 ||
845            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
846   }
847 
848   case AArch64::SUBXrs:
849   case AArch64::SUBSXrs: {
850     unsigned Imm = MI.getOperand(3).getImm();
851     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
852     return ShiftVal == 0 ||
853            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
854   }
855 
856   case AArch64::SUBWrx:
857   case AArch64::SUBXrx:
858   case AArch64::SUBXrx64:
859   case AArch64::SUBSWrx:
860   case AArch64::SUBSXrx:
861   case AArch64::SUBSXrx64: {
862     unsigned Imm = MI.getOperand(3).getImm();
863     switch (AArch64_AM::getArithExtendType(Imm)) {
864     default:
865       return false;
866     case AArch64_AM::UXTB:
867     case AArch64_AM::UXTH:
868     case AArch64_AM::UXTW:
869     case AArch64_AM::UXTX:
870       return AArch64_AM::getArithShiftValue(Imm) == 0;
871     }
872   }
873 
874   case AArch64::LDRBBroW:
875   case AArch64::LDRBBroX:
876   case AArch64::LDRBroW:
877   case AArch64::LDRBroX:
878   case AArch64::LDRDroW:
879   case AArch64::LDRDroX:
880   case AArch64::LDRHHroW:
881   case AArch64::LDRHHroX:
882   case AArch64::LDRHroW:
883   case AArch64::LDRHroX:
884   case AArch64::LDRQroW:
885   case AArch64::LDRQroX:
886   case AArch64::LDRSBWroW:
887   case AArch64::LDRSBWroX:
888   case AArch64::LDRSBXroW:
889   case AArch64::LDRSBXroX:
890   case AArch64::LDRSHWroW:
891   case AArch64::LDRSHWroX:
892   case AArch64::LDRSHXroW:
893   case AArch64::LDRSHXroX:
894   case AArch64::LDRSWroW:
895   case AArch64::LDRSWroX:
896   case AArch64::LDRSroW:
897   case AArch64::LDRSroX:
898   case AArch64::LDRWroW:
899   case AArch64::LDRWroX:
900   case AArch64::LDRXroW:
901   case AArch64::LDRXroX:
902   case AArch64::PRFMroW:
903   case AArch64::PRFMroX:
904   case AArch64::STRBBroW:
905   case AArch64::STRBBroX:
906   case AArch64::STRBroW:
907   case AArch64::STRBroX:
908   case AArch64::STRDroW:
909   case AArch64::STRDroX:
910   case AArch64::STRHHroW:
911   case AArch64::STRHHroX:
912   case AArch64::STRHroW:
913   case AArch64::STRHroX:
914   case AArch64::STRQroW:
915   case AArch64::STRQroX:
916   case AArch64::STRSroW:
917   case AArch64::STRSroX:
918   case AArch64::STRWroW:
919   case AArch64::STRWroX:
920   case AArch64::STRXroW:
921   case AArch64::STRXroX: {
922     unsigned IsSigned = MI.getOperand(3).getImm();
923     return !IsSigned;
924   }
925   }
926 }
927 
928 bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
929   unsigned Opc = MI.getOpcode();
930   switch (Opc) {
931     default:
932       return false;
933     case AArch64::SEH_StackAlloc:
934     case AArch64::SEH_SaveFPLR:
935     case AArch64::SEH_SaveFPLR_X:
936     case AArch64::SEH_SaveReg:
937     case AArch64::SEH_SaveReg_X:
938     case AArch64::SEH_SaveRegP:
939     case AArch64::SEH_SaveRegP_X:
940     case AArch64::SEH_SaveFReg:
941     case AArch64::SEH_SaveFReg_X:
942     case AArch64::SEH_SaveFRegP:
943     case AArch64::SEH_SaveFRegP_X:
944     case AArch64::SEH_SetFP:
945     case AArch64::SEH_AddFP:
946     case AArch64::SEH_Nop:
947     case AArch64::SEH_PrologEnd:
948     case AArch64::SEH_EpilogStart:
949     case AArch64::SEH_EpilogEnd:
950       return true;
951   }
952 }
953 
954 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
955                                              Register &SrcReg, Register &DstReg,
956                                              unsigned &SubIdx) const {
957   switch (MI.getOpcode()) {
958   default:
959     return false;
960   case AArch64::SBFMXri: // aka sxtw
961   case AArch64::UBFMXri: // aka uxtw
962     // Check for the 32 -> 64 bit extension case, these instructions can do
963     // much more.
964     if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
965       return false;
966     // This is a signed or unsigned 32 -> 64 bit extension.
967     SrcReg = MI.getOperand(1).getReg();
968     DstReg = MI.getOperand(0).getReg();
969     SubIdx = AArch64::sub_32;
970     return true;
971   }
972 }
973 
974 bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
975     const MachineInstr &MIa, const MachineInstr &MIb) const {
976   const TargetRegisterInfo *TRI = &getRegisterInfo();
977   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
978   int64_t OffsetA = 0, OffsetB = 0;
979   unsigned WidthA = 0, WidthB = 0;
980   bool OffsetAIsScalable = false, OffsetBIsScalable = false;
981 
982   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
983   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
984 
985   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
986       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
987     return false;
988 
989   // Retrieve the base, offset from the base and width. Width
990   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8).  If
991   // base are identical, and the offset of a lower memory access +
992   // the width doesn't overlap the offset of a higher memory access,
993   // then the memory accesses are different.
994   // If OffsetAIsScalable and OffsetBIsScalable are both true, they
995   // are assumed to have the same scale (vscale).
996   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
997                                    WidthA, TRI) &&
998       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
999                                    WidthB, TRI)) {
1000     if (BaseOpA->isIdenticalTo(*BaseOpB) &&
1001         OffsetAIsScalable == OffsetBIsScalable) {
1002       int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1003       int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1004       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1005       if (LowOffset + LowWidth <= HighOffset)
1006         return true;
1007     }
1008   }
1009   return false;
1010 }
1011 
1012 bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1013                                             const MachineBasicBlock *MBB,
1014                                             const MachineFunction &MF) const {
1015   if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1016     return true;
1017   switch (MI.getOpcode()) {
1018   case AArch64::HINT:
1019     // CSDB hints are scheduling barriers.
1020     if (MI.getOperand(0).getImm() == 0x14)
1021       return true;
1022     break;
1023   case AArch64::DSB:
1024   case AArch64::ISB:
1025     // DSB and ISB also are scheduling barriers.
1026     return true;
1027   default:;
1028   }
1029   return isSEHInstruction(MI);
1030 }
1031 
1032 /// analyzeCompare - For a comparison instruction, return the source registers
1033 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1034 /// Return true if the comparison instruction can be analyzed.
1035 bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1036                                       Register &SrcReg2, int &CmpMask,
1037                                       int &CmpValue) const {
1038   // The first operand can be a frame index where we'd normally expect a
1039   // register.
1040   assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1041   if (!MI.getOperand(1).isReg())
1042     return false;
1043 
1044   switch (MI.getOpcode()) {
1045   default:
1046     break;
1047   case AArch64::SUBSWrr:
1048   case AArch64::SUBSWrs:
1049   case AArch64::SUBSWrx:
1050   case AArch64::SUBSXrr:
1051   case AArch64::SUBSXrs:
1052   case AArch64::SUBSXrx:
1053   case AArch64::ADDSWrr:
1054   case AArch64::ADDSWrs:
1055   case AArch64::ADDSWrx:
1056   case AArch64::ADDSXrr:
1057   case AArch64::ADDSXrs:
1058   case AArch64::ADDSXrx:
1059     // Replace SUBSWrr with SUBWrr if NZCV is not used.
1060     SrcReg = MI.getOperand(1).getReg();
1061     SrcReg2 = MI.getOperand(2).getReg();
1062     CmpMask = ~0;
1063     CmpValue = 0;
1064     return true;
1065   case AArch64::SUBSWri:
1066   case AArch64::ADDSWri:
1067   case AArch64::SUBSXri:
1068   case AArch64::ADDSXri:
1069     SrcReg = MI.getOperand(1).getReg();
1070     SrcReg2 = 0;
1071     CmpMask = ~0;
1072     // FIXME: In order to convert CmpValue to 0 or 1
1073     CmpValue = MI.getOperand(2).getImm() != 0;
1074     return true;
1075   case AArch64::ANDSWri:
1076   case AArch64::ANDSXri:
1077     // ANDS does not use the same encoding scheme as the others xxxS
1078     // instructions.
1079     SrcReg = MI.getOperand(1).getReg();
1080     SrcReg2 = 0;
1081     CmpMask = ~0;
1082     // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
1083     // while the type of CmpValue is int. When converting uint64_t to int,
1084     // the high 32 bits of uint64_t will be lost.
1085     // In fact it causes a bug in spec2006-483.xalancbmk
1086     // CmpValue is only used to compare with zero in OptimizeCompareInstr
1087     CmpValue = AArch64_AM::decodeLogicalImmediate(
1088                    MI.getOperand(2).getImm(),
1089                    MI.getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0;
1090     return true;
1091   }
1092 
1093   return false;
1094 }
1095 
1096 static bool UpdateOperandRegClass(MachineInstr &Instr) {
1097   MachineBasicBlock *MBB = Instr.getParent();
1098   assert(MBB && "Can't get MachineBasicBlock here");
1099   MachineFunction *MF = MBB->getParent();
1100   assert(MF && "Can't get MachineFunction here");
1101   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1102   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1103   MachineRegisterInfo *MRI = &MF->getRegInfo();
1104 
1105   for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1106        ++OpIdx) {
1107     MachineOperand &MO = Instr.getOperand(OpIdx);
1108     const TargetRegisterClass *OpRegCstraints =
1109         Instr.getRegClassConstraint(OpIdx, TII, TRI);
1110 
1111     // If there's no constraint, there's nothing to do.
1112     if (!OpRegCstraints)
1113       continue;
1114     // If the operand is a frame index, there's nothing to do here.
1115     // A frame index operand will resolve correctly during PEI.
1116     if (MO.isFI())
1117       continue;
1118 
1119     assert(MO.isReg() &&
1120            "Operand has register constraints without being a register!");
1121 
1122     Register Reg = MO.getReg();
1123     if (Register::isPhysicalRegister(Reg)) {
1124       if (!OpRegCstraints->contains(Reg))
1125         return false;
1126     } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
1127                !MRI->constrainRegClass(Reg, OpRegCstraints))
1128       return false;
1129   }
1130 
1131   return true;
1132 }
1133 
1134 /// Return the opcode that does not set flags when possible - otherwise
1135 /// return the original opcode. The caller is responsible to do the actual
1136 /// substitution and legality checking.
1137 static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1138   // Don't convert all compare instructions, because for some the zero register
1139   // encoding becomes the sp register.
1140   bool MIDefinesZeroReg = false;
1141   if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR))
1142     MIDefinesZeroReg = true;
1143 
1144   switch (MI.getOpcode()) {
1145   default:
1146     return MI.getOpcode();
1147   case AArch64::ADDSWrr:
1148     return AArch64::ADDWrr;
1149   case AArch64::ADDSWri:
1150     return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1151   case AArch64::ADDSWrs:
1152     return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1153   case AArch64::ADDSWrx:
1154     return AArch64::ADDWrx;
1155   case AArch64::ADDSXrr:
1156     return AArch64::ADDXrr;
1157   case AArch64::ADDSXri:
1158     return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1159   case AArch64::ADDSXrs:
1160     return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1161   case AArch64::ADDSXrx:
1162     return AArch64::ADDXrx;
1163   case AArch64::SUBSWrr:
1164     return AArch64::SUBWrr;
1165   case AArch64::SUBSWri:
1166     return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1167   case AArch64::SUBSWrs:
1168     return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1169   case AArch64::SUBSWrx:
1170     return AArch64::SUBWrx;
1171   case AArch64::SUBSXrr:
1172     return AArch64::SUBXrr;
1173   case AArch64::SUBSXri:
1174     return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1175   case AArch64::SUBSXrs:
1176     return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1177   case AArch64::SUBSXrx:
1178     return AArch64::SUBXrx;
1179   }
1180 }
1181 
1182 enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1183 
1184 /// True when condition flags are accessed (either by writing or reading)
1185 /// on the instruction trace starting at From and ending at To.
1186 ///
1187 /// Note: If From and To are from different blocks it's assumed CC are accessed
1188 ///       on the path.
1189 static bool areCFlagsAccessedBetweenInstrs(
1190     MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1191     const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1192   // Early exit if To is at the beginning of the BB.
1193   if (To == To->getParent()->begin())
1194     return true;
1195 
1196   // Check whether the instructions are in the same basic block
1197   // If not, assume the condition flags might get modified somewhere.
1198   if (To->getParent() != From->getParent())
1199     return true;
1200 
1201   // From must be above To.
1202   assert(std::find_if(++To.getReverse(), To->getParent()->rend(),
1203                       [From](MachineInstr &MI) {
1204                         return MI.getIterator() == From;
1205                       }) != To->getParent()->rend());
1206 
1207   // We iterate backward starting at \p To until we hit \p From.
1208   for (const MachineInstr &Instr :
1209        instructionsWithoutDebug(++To.getReverse(), From.getReverse())) {
1210     if (((AccessToCheck & AK_Write) &&
1211          Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
1212         ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
1213       return true;
1214   }
1215   return false;
1216 }
1217 
1218 /// Try to optimize a compare instruction. A compare instruction is an
1219 /// instruction which produces AArch64::NZCV. It can be truly compare
1220 /// instruction
1221 /// when there are no uses of its destination register.
1222 ///
1223 /// The following steps are tried in order:
1224 /// 1. Convert CmpInstr into an unconditional version.
1225 /// 2. Remove CmpInstr if above there is an instruction producing a needed
1226 ///    condition code or an instruction which can be converted into such an
1227 ///    instruction.
1228 ///    Only comparison with zero is supported.
1229 bool AArch64InstrInfo::optimizeCompareInstr(
1230     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
1231     int CmpValue, const MachineRegisterInfo *MRI) const {
1232   assert(CmpInstr.getParent());
1233   assert(MRI);
1234 
1235   // Replace SUBSWrr with SUBWrr if NZCV is not used.
1236   int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true);
1237   if (DeadNZCVIdx != -1) {
1238     if (CmpInstr.definesRegister(AArch64::WZR) ||
1239         CmpInstr.definesRegister(AArch64::XZR)) {
1240       CmpInstr.eraseFromParent();
1241       return true;
1242     }
1243     unsigned Opc = CmpInstr.getOpcode();
1244     unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
1245     if (NewOpc == Opc)
1246       return false;
1247     const MCInstrDesc &MCID = get(NewOpc);
1248     CmpInstr.setDesc(MCID);
1249     CmpInstr.RemoveOperand(DeadNZCVIdx);
1250     bool succeeded = UpdateOperandRegClass(CmpInstr);
1251     (void)succeeded;
1252     assert(succeeded && "Some operands reg class are incompatible!");
1253     return true;
1254   }
1255 
1256   // Continue only if we have a "ri" where immediate is zero.
1257   // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
1258   // function.
1259   assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
1260   if (CmpValue != 0 || SrcReg2 != 0)
1261     return false;
1262 
1263   // CmpInstr is a Compare instruction if destination register is not used.
1264   if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
1265     return false;
1266 
1267   return substituteCmpToZero(CmpInstr, SrcReg, MRI);
1268 }
1269 
1270 /// Get opcode of S version of Instr.
1271 /// If Instr is S version its opcode is returned.
1272 /// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
1273 /// or we are not interested in it.
1274 static unsigned sForm(MachineInstr &Instr) {
1275   switch (Instr.getOpcode()) {
1276   default:
1277     return AArch64::INSTRUCTION_LIST_END;
1278 
1279   case AArch64::ADDSWrr:
1280   case AArch64::ADDSWri:
1281   case AArch64::ADDSXrr:
1282   case AArch64::ADDSXri:
1283   case AArch64::SUBSWrr:
1284   case AArch64::SUBSWri:
1285   case AArch64::SUBSXrr:
1286   case AArch64::SUBSXri:
1287     return Instr.getOpcode();
1288 
1289   case AArch64::ADDWrr:
1290     return AArch64::ADDSWrr;
1291   case AArch64::ADDWri:
1292     return AArch64::ADDSWri;
1293   case AArch64::ADDXrr:
1294     return AArch64::ADDSXrr;
1295   case AArch64::ADDXri:
1296     return AArch64::ADDSXri;
1297   case AArch64::ADCWr:
1298     return AArch64::ADCSWr;
1299   case AArch64::ADCXr:
1300     return AArch64::ADCSXr;
1301   case AArch64::SUBWrr:
1302     return AArch64::SUBSWrr;
1303   case AArch64::SUBWri:
1304     return AArch64::SUBSWri;
1305   case AArch64::SUBXrr:
1306     return AArch64::SUBSXrr;
1307   case AArch64::SUBXri:
1308     return AArch64::SUBSXri;
1309   case AArch64::SBCWr:
1310     return AArch64::SBCSWr;
1311   case AArch64::SBCXr:
1312     return AArch64::SBCSXr;
1313   case AArch64::ANDWri:
1314     return AArch64::ANDSWri;
1315   case AArch64::ANDXri:
1316     return AArch64::ANDSXri;
1317   }
1318 }
1319 
1320 /// Check if AArch64::NZCV should be alive in successors of MBB.
1321 static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) {
1322   for (auto *BB : MBB->successors())
1323     if (BB->isLiveIn(AArch64::NZCV))
1324       return true;
1325   return false;
1326 }
1327 
1328 namespace {
1329 
1330 struct UsedNZCV {
1331   bool N = false;
1332   bool Z = false;
1333   bool C = false;
1334   bool V = false;
1335 
1336   UsedNZCV() = default;
1337 
1338   UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
1339     this->N |= UsedFlags.N;
1340     this->Z |= UsedFlags.Z;
1341     this->C |= UsedFlags.C;
1342     this->V |= UsedFlags.V;
1343     return *this;
1344   }
1345 };
1346 
1347 } // end anonymous namespace
1348 
1349 /// Find a condition code used by the instruction.
1350 /// Returns AArch64CC::Invalid if either the instruction does not use condition
1351 /// codes or we don't optimize CmpInstr in the presence of such instructions.
1352 static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
1353   switch (Instr.getOpcode()) {
1354   default:
1355     return AArch64CC::Invalid;
1356 
1357   case AArch64::Bcc: {
1358     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1359     assert(Idx >= 2);
1360     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 2).getImm());
1361   }
1362 
1363   case AArch64::CSINVWr:
1364   case AArch64::CSINVXr:
1365   case AArch64::CSINCWr:
1366   case AArch64::CSINCXr:
1367   case AArch64::CSELWr:
1368   case AArch64::CSELXr:
1369   case AArch64::CSNEGWr:
1370   case AArch64::CSNEGXr:
1371   case AArch64::FCSELSrrr:
1372   case AArch64::FCSELDrrr: {
1373     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1374     assert(Idx >= 1);
1375     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 1).getImm());
1376   }
1377   }
1378 }
1379 
1380 static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
1381   assert(CC != AArch64CC::Invalid);
1382   UsedNZCV UsedFlags;
1383   switch (CC) {
1384   default:
1385     break;
1386 
1387   case AArch64CC::EQ: // Z set
1388   case AArch64CC::NE: // Z clear
1389     UsedFlags.Z = true;
1390     break;
1391 
1392   case AArch64CC::HI: // Z clear and C set
1393   case AArch64CC::LS: // Z set   or  C clear
1394     UsedFlags.Z = true;
1395     LLVM_FALLTHROUGH;
1396   case AArch64CC::HS: // C set
1397   case AArch64CC::LO: // C clear
1398     UsedFlags.C = true;
1399     break;
1400 
1401   case AArch64CC::MI: // N set
1402   case AArch64CC::PL: // N clear
1403     UsedFlags.N = true;
1404     break;
1405 
1406   case AArch64CC::VS: // V set
1407   case AArch64CC::VC: // V clear
1408     UsedFlags.V = true;
1409     break;
1410 
1411   case AArch64CC::GT: // Z clear, N and V the same
1412   case AArch64CC::LE: // Z set,   N and V differ
1413     UsedFlags.Z = true;
1414     LLVM_FALLTHROUGH;
1415   case AArch64CC::GE: // N and V the same
1416   case AArch64CC::LT: // N and V differ
1417     UsedFlags.N = true;
1418     UsedFlags.V = true;
1419     break;
1420   }
1421   return UsedFlags;
1422 }
1423 
1424 static bool isADDSRegImm(unsigned Opcode) {
1425   return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1426 }
1427 
1428 static bool isSUBSRegImm(unsigned Opcode) {
1429   return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1430 }
1431 
1432 /// Check if CmpInstr can be substituted by MI.
1433 ///
1434 /// CmpInstr can be substituted:
1435 /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1436 /// - and, MI and CmpInstr are from the same MachineBB
1437 /// - and, condition flags are not alive in successors of the CmpInstr parent
1438 /// - and, if MI opcode is the S form there must be no defs of flags between
1439 ///        MI and CmpInstr
1440 ///        or if MI opcode is not the S form there must be neither defs of flags
1441 ///        nor uses of flags between MI and CmpInstr.
1442 /// - and  C/V flags are not used after CmpInstr
1443 static bool canInstrSubstituteCmpInstr(MachineInstr *MI, MachineInstr *CmpInstr,
1444                                        const TargetRegisterInfo *TRI) {
1445   assert(MI);
1446   assert(sForm(*MI) != AArch64::INSTRUCTION_LIST_END);
1447   assert(CmpInstr);
1448 
1449   const unsigned CmpOpcode = CmpInstr->getOpcode();
1450   if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
1451     return false;
1452 
1453   if (MI->getParent() != CmpInstr->getParent())
1454     return false;
1455 
1456   if (areCFlagsAliveInSuccessors(CmpInstr->getParent()))
1457     return false;
1458 
1459   AccessKind AccessToCheck = AK_Write;
1460   if (sForm(*MI) != MI->getOpcode())
1461     AccessToCheck = AK_All;
1462   if (areCFlagsAccessedBetweenInstrs(MI, CmpInstr, TRI, AccessToCheck))
1463     return false;
1464 
1465   UsedNZCV NZCVUsedAfterCmp;
1466   for (const MachineInstr &Instr :
1467        instructionsWithoutDebug(std::next(CmpInstr->getIterator()),
1468                                 CmpInstr->getParent()->instr_end())) {
1469     if (Instr.readsRegister(AArch64::NZCV, TRI)) {
1470       AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
1471       if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
1472         return false;
1473       NZCVUsedAfterCmp |= getUsedNZCV(CC);
1474     }
1475 
1476     if (Instr.modifiesRegister(AArch64::NZCV, TRI))
1477       break;
1478   }
1479 
1480   return !NZCVUsedAfterCmp.C && !NZCVUsedAfterCmp.V;
1481 }
1482 
1483 /// Substitute an instruction comparing to zero with another instruction
1484 /// which produces needed condition flags.
1485 ///
1486 /// Return true on success.
1487 bool AArch64InstrInfo::substituteCmpToZero(
1488     MachineInstr &CmpInstr, unsigned SrcReg,
1489     const MachineRegisterInfo *MRI) const {
1490   assert(MRI);
1491   // Get the unique definition of SrcReg.
1492   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1493   if (!MI)
1494     return false;
1495 
1496   const TargetRegisterInfo *TRI = &getRegisterInfo();
1497 
1498   unsigned NewOpc = sForm(*MI);
1499   if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1500     return false;
1501 
1502   if (!canInstrSubstituteCmpInstr(MI, &CmpInstr, TRI))
1503     return false;
1504 
1505   // Update the instruction to set NZCV.
1506   MI->setDesc(get(NewOpc));
1507   CmpInstr.eraseFromParent();
1508   bool succeeded = UpdateOperandRegClass(*MI);
1509   (void)succeeded;
1510   assert(succeeded && "Some operands reg class are incompatible!");
1511   MI->addRegisterDefined(AArch64::NZCV, TRI);
1512   return true;
1513 }
1514 
1515 bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1516   if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1517       MI.getOpcode() != AArch64::CATCHRET)
1518     return false;
1519 
1520   MachineBasicBlock &MBB = *MI.getParent();
1521   auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1522   auto TRI = Subtarget.getRegisterInfo();
1523   DebugLoc DL = MI.getDebugLoc();
1524 
1525   if (MI.getOpcode() == AArch64::CATCHRET) {
1526     // Skip to the first instruction before the epilog.
1527     const TargetInstrInfo *TII =
1528       MBB.getParent()->getSubtarget().getInstrInfo();
1529     MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
1530     auto MBBI = MachineBasicBlock::iterator(MI);
1531     MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
1532     while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
1533            FirstEpilogSEH != MBB.begin())
1534       FirstEpilogSEH = std::prev(FirstEpilogSEH);
1535     if (FirstEpilogSEH != MBB.begin())
1536       FirstEpilogSEH = std::next(FirstEpilogSEH);
1537     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
1538         .addReg(AArch64::X0, RegState::Define)
1539         .addMBB(TargetMBB);
1540     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
1541         .addReg(AArch64::X0, RegState::Define)
1542         .addReg(AArch64::X0)
1543         .addMBB(TargetMBB)
1544         .addImm(0);
1545     return true;
1546   }
1547 
1548   Register Reg = MI.getOperand(0).getReg();
1549   const GlobalValue *GV =
1550       cast<GlobalValue>((*MI.memoperands_begin())->getValue());
1551   const TargetMachine &TM = MBB.getParent()->getTarget();
1552   unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
1553   const unsigned char MO_NC = AArch64II::MO_NC;
1554 
1555   if ((OpFlags & AArch64II::MO_GOT) != 0) {
1556     BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
1557         .addGlobalAddress(GV, 0, OpFlags);
1558     if (Subtarget.isTargetILP32()) {
1559       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1560       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1561           .addDef(Reg32, RegState::Dead)
1562           .addUse(Reg, RegState::Kill)
1563           .addImm(0)
1564           .addMemOperand(*MI.memoperands_begin())
1565           .addDef(Reg, RegState::Implicit);
1566     } else {
1567       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1568           .addReg(Reg, RegState::Kill)
1569           .addImm(0)
1570           .addMemOperand(*MI.memoperands_begin());
1571     }
1572   } else if (TM.getCodeModel() == CodeModel::Large) {
1573     assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?");
1574     BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
1575         .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
1576         .addImm(0);
1577     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1578         .addReg(Reg, RegState::Kill)
1579         .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
1580         .addImm(16);
1581     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1582         .addReg(Reg, RegState::Kill)
1583         .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
1584         .addImm(32);
1585     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1586         .addReg(Reg, RegState::Kill)
1587         .addGlobalAddress(GV, 0, AArch64II::MO_G3)
1588         .addImm(48);
1589     BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1590         .addReg(Reg, RegState::Kill)
1591         .addImm(0)
1592         .addMemOperand(*MI.memoperands_begin());
1593   } else if (TM.getCodeModel() == CodeModel::Tiny) {
1594     BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg)
1595         .addGlobalAddress(GV, 0, OpFlags);
1596   } else {
1597     BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
1598         .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
1599     unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
1600     if (Subtarget.isTargetILP32()) {
1601       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1602       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1603           .addDef(Reg32, RegState::Dead)
1604           .addUse(Reg, RegState::Kill)
1605           .addGlobalAddress(GV, 0, LoFlags)
1606           .addMemOperand(*MI.memoperands_begin())
1607           .addDef(Reg, RegState::Implicit);
1608     } else {
1609       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1610           .addReg(Reg, RegState::Kill)
1611           .addGlobalAddress(GV, 0, LoFlags)
1612           .addMemOperand(*MI.memoperands_begin());
1613     }
1614   }
1615 
1616   MBB.erase(MI);
1617 
1618   return true;
1619 }
1620 
1621 // Return true if this instruction simply sets its single destination register
1622 // to zero. This is equivalent to a register rename of the zero-register.
1623 bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
1624   switch (MI.getOpcode()) {
1625   default:
1626     break;
1627   case AArch64::MOVZWi:
1628   case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1629     if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
1630       assert(MI.getDesc().getNumOperands() == 3 &&
1631              MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1632       return true;
1633     }
1634     break;
1635   case AArch64::ANDWri: // and Rd, Rzr, #imm
1636     return MI.getOperand(1).getReg() == AArch64::WZR;
1637   case AArch64::ANDXri:
1638     return MI.getOperand(1).getReg() == AArch64::XZR;
1639   case TargetOpcode::COPY:
1640     return MI.getOperand(1).getReg() == AArch64::WZR;
1641   }
1642   return false;
1643 }
1644 
1645 // Return true if this instruction simply renames a general register without
1646 // modifying bits.
1647 bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
1648   switch (MI.getOpcode()) {
1649   default:
1650     break;
1651   case TargetOpcode::COPY: {
1652     // GPR32 copies will by lowered to ORRXrs
1653     Register DstReg = MI.getOperand(0).getReg();
1654     return (AArch64::GPR32RegClass.contains(DstReg) ||
1655             AArch64::GPR64RegClass.contains(DstReg));
1656   }
1657   case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1658     if (MI.getOperand(1).getReg() == AArch64::XZR) {
1659       assert(MI.getDesc().getNumOperands() == 4 &&
1660              MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1661       return true;
1662     }
1663     break;
1664   case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1665     if (MI.getOperand(2).getImm() == 0) {
1666       assert(MI.getDesc().getNumOperands() == 4 &&
1667              MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1668       return true;
1669     }
1670     break;
1671   }
1672   return false;
1673 }
1674 
1675 // Return true if this instruction simply renames a general register without
1676 // modifying bits.
1677 bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
1678   switch (MI.getOpcode()) {
1679   default:
1680     break;
1681   case TargetOpcode::COPY: {
1682     // FPR64 copies will by lowered to ORR.16b
1683     Register DstReg = MI.getOperand(0).getReg();
1684     return (AArch64::FPR64RegClass.contains(DstReg) ||
1685             AArch64::FPR128RegClass.contains(DstReg));
1686   }
1687   case AArch64::ORRv16i8:
1688     if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
1689       assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
1690              "invalid ORRv16i8 operands");
1691       return true;
1692     }
1693     break;
1694   }
1695   return false;
1696 }
1697 
1698 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1699                                                int &FrameIndex) const {
1700   switch (MI.getOpcode()) {
1701   default:
1702     break;
1703   case AArch64::LDRWui:
1704   case AArch64::LDRXui:
1705   case AArch64::LDRBui:
1706   case AArch64::LDRHui:
1707   case AArch64::LDRSui:
1708   case AArch64::LDRDui:
1709   case AArch64::LDRQui:
1710     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1711         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1712       FrameIndex = MI.getOperand(1).getIndex();
1713       return MI.getOperand(0).getReg();
1714     }
1715     break;
1716   }
1717 
1718   return 0;
1719 }
1720 
1721 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1722                                               int &FrameIndex) const {
1723   switch (MI.getOpcode()) {
1724   default:
1725     break;
1726   case AArch64::STRWui:
1727   case AArch64::STRXui:
1728   case AArch64::STRBui:
1729   case AArch64::STRHui:
1730   case AArch64::STRSui:
1731   case AArch64::STRDui:
1732   case AArch64::STRQui:
1733   case AArch64::LDR_PXI:
1734   case AArch64::STR_PXI:
1735     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1736         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1737       FrameIndex = MI.getOperand(1).getIndex();
1738       return MI.getOperand(0).getReg();
1739     }
1740     break;
1741   }
1742   return 0;
1743 }
1744 
1745 /// Check all MachineMemOperands for a hint to suppress pairing.
1746 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
1747   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1748     return MMO->getFlags() & MOSuppressPair;
1749   });
1750 }
1751 
1752 /// Set a flag on the first MachineMemOperand to suppress pairing.
1753 void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
1754   if (MI.memoperands_empty())
1755     return;
1756   (*MI.memoperands_begin())->setFlags(MOSuppressPair);
1757 }
1758 
1759 /// Check all MachineMemOperands for a hint that the load/store is strided.
1760 bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
1761   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1762     return MMO->getFlags() & MOStridedAccess;
1763   });
1764 }
1765 
1766 bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) {
1767   switch (Opc) {
1768   default:
1769     return false;
1770   case AArch64::STURSi:
1771   case AArch64::STURDi:
1772   case AArch64::STURQi:
1773   case AArch64::STURBBi:
1774   case AArch64::STURHHi:
1775   case AArch64::STURWi:
1776   case AArch64::STURXi:
1777   case AArch64::LDURSi:
1778   case AArch64::LDURDi:
1779   case AArch64::LDURQi:
1780   case AArch64::LDURWi:
1781   case AArch64::LDURXi:
1782   case AArch64::LDURSWi:
1783   case AArch64::LDURHHi:
1784   case AArch64::LDURBBi:
1785   case AArch64::LDURSBWi:
1786   case AArch64::LDURSHWi:
1787     return true;
1788   }
1789 }
1790 
1791 Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
1792   switch (Opc) {
1793   default: return {};
1794   case AArch64::PRFMui: return AArch64::PRFUMi;
1795   case AArch64::LDRXui: return AArch64::LDURXi;
1796   case AArch64::LDRWui: return AArch64::LDURWi;
1797   case AArch64::LDRBui: return AArch64::LDURBi;
1798   case AArch64::LDRHui: return AArch64::LDURHi;
1799   case AArch64::LDRSui: return AArch64::LDURSi;
1800   case AArch64::LDRDui: return AArch64::LDURDi;
1801   case AArch64::LDRQui: return AArch64::LDURQi;
1802   case AArch64::LDRBBui: return AArch64::LDURBBi;
1803   case AArch64::LDRHHui: return AArch64::LDURHHi;
1804   case AArch64::LDRSBXui: return AArch64::LDURSBXi;
1805   case AArch64::LDRSBWui: return AArch64::LDURSBWi;
1806   case AArch64::LDRSHXui: return AArch64::LDURSHXi;
1807   case AArch64::LDRSHWui: return AArch64::LDURSHWi;
1808   case AArch64::LDRSWui: return AArch64::LDURSWi;
1809   case AArch64::STRXui: return AArch64::STURXi;
1810   case AArch64::STRWui: return AArch64::STURWi;
1811   case AArch64::STRBui: return AArch64::STURBi;
1812   case AArch64::STRHui: return AArch64::STURHi;
1813   case AArch64::STRSui: return AArch64::STURSi;
1814   case AArch64::STRDui: return AArch64::STURDi;
1815   case AArch64::STRQui: return AArch64::STURQi;
1816   case AArch64::STRBBui: return AArch64::STURBBi;
1817   case AArch64::STRHHui: return AArch64::STURHHi;
1818   }
1819 }
1820 
1821 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
1822   switch (Opc) {
1823   default:
1824     return 2;
1825   case AArch64::LDPXi:
1826   case AArch64::LDPDi:
1827   case AArch64::STPXi:
1828   case AArch64::STPDi:
1829   case AArch64::LDNPXi:
1830   case AArch64::LDNPDi:
1831   case AArch64::STNPXi:
1832   case AArch64::STNPDi:
1833   case AArch64::LDPQi:
1834   case AArch64::STPQi:
1835   case AArch64::LDNPQi:
1836   case AArch64::STNPQi:
1837   case AArch64::LDPWi:
1838   case AArch64::LDPSi:
1839   case AArch64::STPWi:
1840   case AArch64::STPSi:
1841   case AArch64::LDNPWi:
1842   case AArch64::LDNPSi:
1843   case AArch64::STNPWi:
1844   case AArch64::STNPSi:
1845   case AArch64::LDG:
1846   case AArch64::STGPi:
1847   case AArch64::LD1B_IMM:
1848   case AArch64::LD1H_IMM:
1849   case AArch64::LD1W_IMM:
1850   case AArch64::LD1D_IMM:
1851   case AArch64::ST1B_IMM:
1852   case AArch64::ST1H_IMM:
1853   case AArch64::ST1W_IMM:
1854   case AArch64::ST1D_IMM:
1855   case AArch64::LD1B_H_IMM:
1856   case AArch64::LD1SB_H_IMM:
1857   case AArch64::LD1H_S_IMM:
1858   case AArch64::LD1SH_S_IMM:
1859   case AArch64::LD1W_D_IMM:
1860   case AArch64::LD1SW_D_IMM:
1861   case AArch64::ST1B_H_IMM:
1862   case AArch64::ST1H_S_IMM:
1863   case AArch64::ST1W_D_IMM:
1864   case AArch64::LD1B_S_IMM:
1865   case AArch64::LD1SB_S_IMM:
1866   case AArch64::LD1H_D_IMM:
1867   case AArch64::LD1SH_D_IMM:
1868   case AArch64::ST1B_S_IMM:
1869   case AArch64::ST1H_D_IMM:
1870   case AArch64::LD1B_D_IMM:
1871   case AArch64::LD1SB_D_IMM:
1872   case AArch64::ST1B_D_IMM:
1873     return 3;
1874   case AArch64::ADDG:
1875   case AArch64::STGOffset:
1876   case AArch64::LDR_PXI:
1877   case AArch64::STR_PXI:
1878     return 2;
1879   }
1880 }
1881 
1882 bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
1883   switch (MI.getOpcode()) {
1884   default:
1885     return false;
1886   // Scaled instructions.
1887   case AArch64::STRSui:
1888   case AArch64::STRDui:
1889   case AArch64::STRQui:
1890   case AArch64::STRXui:
1891   case AArch64::STRWui:
1892   case AArch64::LDRSui:
1893   case AArch64::LDRDui:
1894   case AArch64::LDRQui:
1895   case AArch64::LDRXui:
1896   case AArch64::LDRWui:
1897   case AArch64::LDRSWui:
1898   // Unscaled instructions.
1899   case AArch64::STURSi:
1900   case AArch64::STURDi:
1901   case AArch64::STURQi:
1902   case AArch64::STURWi:
1903   case AArch64::STURXi:
1904   case AArch64::LDURSi:
1905   case AArch64::LDURDi:
1906   case AArch64::LDURQi:
1907   case AArch64::LDURWi:
1908   case AArch64::LDURXi:
1909   case AArch64::LDURSWi:
1910     return true;
1911   }
1912 }
1913 
1914 unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc,
1915                                                    bool &Is64Bit) {
1916   switch (Opc) {
1917   default:
1918     llvm_unreachable("Opcode has no flag setting equivalent!");
1919   // 32-bit cases:
1920   case AArch64::ADDWri:
1921     Is64Bit = false;
1922     return AArch64::ADDSWri;
1923   case AArch64::ADDWrr:
1924     Is64Bit = false;
1925     return AArch64::ADDSWrr;
1926   case AArch64::ADDWrs:
1927     Is64Bit = false;
1928     return AArch64::ADDSWrs;
1929   case AArch64::ADDWrx:
1930     Is64Bit = false;
1931     return AArch64::ADDSWrx;
1932   case AArch64::ANDWri:
1933     Is64Bit = false;
1934     return AArch64::ANDSWri;
1935   case AArch64::ANDWrr:
1936     Is64Bit = false;
1937     return AArch64::ANDSWrr;
1938   case AArch64::ANDWrs:
1939     Is64Bit = false;
1940     return AArch64::ANDSWrs;
1941   case AArch64::BICWrr:
1942     Is64Bit = false;
1943     return AArch64::BICSWrr;
1944   case AArch64::BICWrs:
1945     Is64Bit = false;
1946     return AArch64::BICSWrs;
1947   case AArch64::SUBWri:
1948     Is64Bit = false;
1949     return AArch64::SUBSWri;
1950   case AArch64::SUBWrr:
1951     Is64Bit = false;
1952     return AArch64::SUBSWrr;
1953   case AArch64::SUBWrs:
1954     Is64Bit = false;
1955     return AArch64::SUBSWrs;
1956   case AArch64::SUBWrx:
1957     Is64Bit = false;
1958     return AArch64::SUBSWrx;
1959   // 64-bit cases:
1960   case AArch64::ADDXri:
1961     Is64Bit = true;
1962     return AArch64::ADDSXri;
1963   case AArch64::ADDXrr:
1964     Is64Bit = true;
1965     return AArch64::ADDSXrr;
1966   case AArch64::ADDXrs:
1967     Is64Bit = true;
1968     return AArch64::ADDSXrs;
1969   case AArch64::ADDXrx:
1970     Is64Bit = true;
1971     return AArch64::ADDSXrx;
1972   case AArch64::ANDXri:
1973     Is64Bit = true;
1974     return AArch64::ANDSXri;
1975   case AArch64::ANDXrr:
1976     Is64Bit = true;
1977     return AArch64::ANDSXrr;
1978   case AArch64::ANDXrs:
1979     Is64Bit = true;
1980     return AArch64::ANDSXrs;
1981   case AArch64::BICXrr:
1982     Is64Bit = true;
1983     return AArch64::BICSXrr;
1984   case AArch64::BICXrs:
1985     Is64Bit = true;
1986     return AArch64::BICSXrs;
1987   case AArch64::SUBXri:
1988     Is64Bit = true;
1989     return AArch64::SUBSXri;
1990   case AArch64::SUBXrr:
1991     Is64Bit = true;
1992     return AArch64::SUBSXrr;
1993   case AArch64::SUBXrs:
1994     Is64Bit = true;
1995     return AArch64::SUBSXrs;
1996   case AArch64::SUBXrx:
1997     Is64Bit = true;
1998     return AArch64::SUBSXrx;
1999   }
2000 }
2001 
2002 // Is this a candidate for ld/st merging or pairing?  For example, we don't
2003 // touch volatiles or load/stores that have a hint to avoid pair formation.
2004 bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
2005   // If this is a volatile load/store, don't mess with it.
2006   if (MI.hasOrderedMemoryRef())
2007     return false;
2008 
2009   // Make sure this is a reg/fi+imm (as opposed to an address reloc).
2010   assert((MI.getOperand(1).isReg() || MI.getOperand(1).isFI()) &&
2011          "Expected a reg or frame index operand.");
2012   if (!MI.getOperand(2).isImm())
2013     return false;
2014 
2015   // Can't merge/pair if the instruction modifies the base register.
2016   // e.g., ldr x0, [x0]
2017   // This case will never occur with an FI base.
2018   if (MI.getOperand(1).isReg()) {
2019     Register BaseReg = MI.getOperand(1).getReg();
2020     const TargetRegisterInfo *TRI = &getRegisterInfo();
2021     if (MI.modifiesRegister(BaseReg, TRI))
2022       return false;
2023   }
2024 
2025   // Check if this load/store has a hint to avoid pair formation.
2026   // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
2027   if (isLdStPairSuppressed(MI))
2028     return false;
2029 
2030   // Do not pair any callee-save store/reload instructions in the
2031   // prologue/epilogue if the CFI information encoded the operations as separate
2032   // instructions, as that will cause the size of the actual prologue to mismatch
2033   // with the prologue size recorded in the Windows CFI.
2034   const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo();
2035   bool NeedsWinCFI = MAI->usesWindowsCFI() &&
2036                      MI.getMF()->getFunction().needsUnwindTableEntry();
2037   if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
2038                       MI.getFlag(MachineInstr::FrameDestroy)))
2039     return false;
2040 
2041   // On some CPUs quad load/store pairs are slower than two single load/stores.
2042   if (Subtarget.isPaired128Slow()) {
2043     switch (MI.getOpcode()) {
2044     default:
2045       break;
2046     case AArch64::LDURQi:
2047     case AArch64::STURQi:
2048     case AArch64::LDRQui:
2049     case AArch64::STRQui:
2050       return false;
2051     }
2052   }
2053 
2054   return true;
2055 }
2056 
2057 bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
2058     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2059     int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2060     const TargetRegisterInfo *TRI) const {
2061   if (!LdSt.mayLoadOrStore())
2062     return false;
2063 
2064   const MachineOperand *BaseOp;
2065   if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
2066                                     Width, TRI))
2067     return false;
2068   BaseOps.push_back(BaseOp);
2069   return true;
2070 }
2071 
2072 bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
2073     const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
2074     bool &OffsetIsScalable, unsigned &Width,
2075     const TargetRegisterInfo *TRI) const {
2076   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2077   // Handle only loads/stores with base register followed by immediate offset.
2078   if (LdSt.getNumExplicitOperands() == 3) {
2079     // Non-paired instruction (e.g., ldr x1, [x0, #8]).
2080     if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2081         !LdSt.getOperand(2).isImm())
2082       return false;
2083   } else if (LdSt.getNumExplicitOperands() == 4) {
2084     // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
2085     if (!LdSt.getOperand(1).isReg() ||
2086         (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
2087         !LdSt.getOperand(3).isImm())
2088       return false;
2089   } else
2090     return false;
2091 
2092   // Get the scaling factor for the instruction and set the width for the
2093   // instruction.
2094   TypeSize Scale(0U, false);
2095   int64_t Dummy1, Dummy2;
2096 
2097   // If this returns false, then it's an instruction we don't want to handle.
2098   if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
2099     return false;
2100 
2101   // Compute the offset. Offset is calculated as the immediate operand
2102   // multiplied by the scaling factor. Unscaled instructions have scaling factor
2103   // set to 1.
2104   if (LdSt.getNumExplicitOperands() == 3) {
2105     BaseOp = &LdSt.getOperand(1);
2106     Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinSize();
2107   } else {
2108     assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
2109     BaseOp = &LdSt.getOperand(2);
2110     Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinSize();
2111   }
2112   OffsetIsScalable = Scale.isScalable();
2113 
2114   if (!BaseOp->isReg() && !BaseOp->isFI())
2115     return false;
2116 
2117   return true;
2118 }
2119 
2120 MachineOperand &
2121 AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
2122   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2123   MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
2124   assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
2125   return OfsOp;
2126 }
2127 
2128 bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
2129                                     unsigned &Width, int64_t &MinOffset,
2130                                     int64_t &MaxOffset) {
2131   const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
2132   switch (Opcode) {
2133   // Not a memory operation or something we want to handle.
2134   default:
2135     Scale = TypeSize::Fixed(0);
2136     Width = 0;
2137     MinOffset = MaxOffset = 0;
2138     return false;
2139   case AArch64::STRWpost:
2140   case AArch64::LDRWpost:
2141     Width = 32;
2142     Scale = TypeSize::Fixed(4);
2143     MinOffset = -256;
2144     MaxOffset = 255;
2145     break;
2146   case AArch64::LDURQi:
2147   case AArch64::STURQi:
2148     Width = 16;
2149     Scale = TypeSize::Fixed(1);
2150     MinOffset = -256;
2151     MaxOffset = 255;
2152     break;
2153   case AArch64::PRFUMi:
2154   case AArch64::LDURXi:
2155   case AArch64::LDURDi:
2156   case AArch64::STURXi:
2157   case AArch64::STURDi:
2158     Width = 8;
2159     Scale = TypeSize::Fixed(1);
2160     MinOffset = -256;
2161     MaxOffset = 255;
2162     break;
2163   case AArch64::LDURWi:
2164   case AArch64::LDURSi:
2165   case AArch64::LDURSWi:
2166   case AArch64::STURWi:
2167   case AArch64::STURSi:
2168     Width = 4;
2169     Scale = TypeSize::Fixed(1);
2170     MinOffset = -256;
2171     MaxOffset = 255;
2172     break;
2173   case AArch64::LDURHi:
2174   case AArch64::LDURHHi:
2175   case AArch64::LDURSHXi:
2176   case AArch64::LDURSHWi:
2177   case AArch64::STURHi:
2178   case AArch64::STURHHi:
2179     Width = 2;
2180     Scale = TypeSize::Fixed(1);
2181     MinOffset = -256;
2182     MaxOffset = 255;
2183     break;
2184   case AArch64::LDURBi:
2185   case AArch64::LDURBBi:
2186   case AArch64::LDURSBXi:
2187   case AArch64::LDURSBWi:
2188   case AArch64::STURBi:
2189   case AArch64::STURBBi:
2190     Width = 1;
2191     Scale = TypeSize::Fixed(1);
2192     MinOffset = -256;
2193     MaxOffset = 255;
2194     break;
2195   case AArch64::LDPQi:
2196   case AArch64::LDNPQi:
2197   case AArch64::STPQi:
2198   case AArch64::STNPQi:
2199     Scale = TypeSize::Fixed(16);
2200     Width = 32;
2201     MinOffset = -64;
2202     MaxOffset = 63;
2203     break;
2204   case AArch64::LDRQui:
2205   case AArch64::STRQui:
2206     Scale = TypeSize::Fixed(16);
2207     Width = 16;
2208     MinOffset = 0;
2209     MaxOffset = 4095;
2210     break;
2211   case AArch64::LDPXi:
2212   case AArch64::LDPDi:
2213   case AArch64::LDNPXi:
2214   case AArch64::LDNPDi:
2215   case AArch64::STPXi:
2216   case AArch64::STPDi:
2217   case AArch64::STNPXi:
2218   case AArch64::STNPDi:
2219     Scale = TypeSize::Fixed(8);
2220     Width = 16;
2221     MinOffset = -64;
2222     MaxOffset = 63;
2223     break;
2224   case AArch64::PRFMui:
2225   case AArch64::LDRXui:
2226   case AArch64::LDRDui:
2227   case AArch64::STRXui:
2228   case AArch64::STRDui:
2229     Scale = TypeSize::Fixed(8);
2230     Width = 8;
2231     MinOffset = 0;
2232     MaxOffset = 4095;
2233     break;
2234   case AArch64::LDPWi:
2235   case AArch64::LDPSi:
2236   case AArch64::LDNPWi:
2237   case AArch64::LDNPSi:
2238   case AArch64::STPWi:
2239   case AArch64::STPSi:
2240   case AArch64::STNPWi:
2241   case AArch64::STNPSi:
2242     Scale = TypeSize::Fixed(4);
2243     Width = 8;
2244     MinOffset = -64;
2245     MaxOffset = 63;
2246     break;
2247   case AArch64::LDRWui:
2248   case AArch64::LDRSui:
2249   case AArch64::LDRSWui:
2250   case AArch64::STRWui:
2251   case AArch64::STRSui:
2252     Scale = TypeSize::Fixed(4);
2253     Width = 4;
2254     MinOffset = 0;
2255     MaxOffset = 4095;
2256     break;
2257   case AArch64::LDRHui:
2258   case AArch64::LDRHHui:
2259   case AArch64::LDRSHWui:
2260   case AArch64::LDRSHXui:
2261   case AArch64::STRHui:
2262   case AArch64::STRHHui:
2263     Scale = TypeSize::Fixed(2);
2264     Width = 2;
2265     MinOffset = 0;
2266     MaxOffset = 4095;
2267     break;
2268   case AArch64::LDRBui:
2269   case AArch64::LDRBBui:
2270   case AArch64::LDRSBWui:
2271   case AArch64::LDRSBXui:
2272   case AArch64::STRBui:
2273   case AArch64::STRBBui:
2274     Scale = TypeSize::Fixed(1);
2275     Width = 1;
2276     MinOffset = 0;
2277     MaxOffset = 4095;
2278     break;
2279   case AArch64::ADDG:
2280     Scale = TypeSize::Fixed(16);
2281     Width = 0;
2282     MinOffset = 0;
2283     MaxOffset = 63;
2284     break;
2285   case AArch64::TAGPstack:
2286     Scale = TypeSize::Fixed(16);
2287     Width = 0;
2288     // TAGP with a negative offset turns into SUBP, which has a maximum offset
2289     // of 63 (not 64!).
2290     MinOffset = -63;
2291     MaxOffset = 63;
2292     break;
2293   case AArch64::LDG:
2294   case AArch64::STGOffset:
2295   case AArch64::STZGOffset:
2296     Scale = TypeSize::Fixed(16);
2297     Width = 16;
2298     MinOffset = -256;
2299     MaxOffset = 255;
2300     break;
2301   case AArch64::STR_ZZZZXI:
2302   case AArch64::LDR_ZZZZXI:
2303     Scale = TypeSize::Scalable(16);
2304     Width = SVEMaxBytesPerVector * 4;
2305     MinOffset = -256;
2306     MaxOffset = 252;
2307     break;
2308   case AArch64::STR_ZZZXI:
2309   case AArch64::LDR_ZZZXI:
2310     Scale = TypeSize::Scalable(16);
2311     Width = SVEMaxBytesPerVector * 3;
2312     MinOffset = -256;
2313     MaxOffset = 253;
2314     break;
2315   case AArch64::STR_ZZXI:
2316   case AArch64::LDR_ZZXI:
2317     Scale = TypeSize::Scalable(16);
2318     Width = SVEMaxBytesPerVector * 2;
2319     MinOffset = -256;
2320     MaxOffset = 254;
2321     break;
2322   case AArch64::LDR_PXI:
2323   case AArch64::STR_PXI:
2324     Scale = TypeSize::Scalable(2);
2325     Width = SVEMaxBytesPerVector / 8;
2326     MinOffset = -256;
2327     MaxOffset = 255;
2328     break;
2329   case AArch64::LDR_ZXI:
2330   case AArch64::STR_ZXI:
2331     Scale = TypeSize::Scalable(16);
2332     Width = SVEMaxBytesPerVector;
2333     MinOffset = -256;
2334     MaxOffset = 255;
2335     break;
2336   case AArch64::LD1B_IMM:
2337   case AArch64::LD1H_IMM:
2338   case AArch64::LD1W_IMM:
2339   case AArch64::LD1D_IMM:
2340   case AArch64::ST1B_IMM:
2341   case AArch64::ST1H_IMM:
2342   case AArch64::ST1W_IMM:
2343   case AArch64::ST1D_IMM:
2344     // A full vectors worth of data
2345     // Width = mbytes * elements
2346     Scale = TypeSize::Scalable(16);
2347     Width = SVEMaxBytesPerVector;
2348     MinOffset = -8;
2349     MaxOffset = 7;
2350     break;
2351   case AArch64::LD1B_H_IMM:
2352   case AArch64::LD1SB_H_IMM:
2353   case AArch64::LD1H_S_IMM:
2354   case AArch64::LD1SH_S_IMM:
2355   case AArch64::LD1W_D_IMM:
2356   case AArch64::LD1SW_D_IMM:
2357   case AArch64::ST1B_H_IMM:
2358   case AArch64::ST1H_S_IMM:
2359   case AArch64::ST1W_D_IMM:
2360     // A half vector worth of data
2361     // Width = mbytes * elements
2362     Scale = TypeSize::Scalable(8);
2363     Width = SVEMaxBytesPerVector / 2;
2364     MinOffset = -8;
2365     MaxOffset = 7;
2366     break;
2367   case AArch64::LD1B_S_IMM:
2368   case AArch64::LD1SB_S_IMM:
2369   case AArch64::LD1H_D_IMM:
2370   case AArch64::LD1SH_D_IMM:
2371   case AArch64::ST1B_S_IMM:
2372   case AArch64::ST1H_D_IMM:
2373     // A quarter vector worth of data
2374     // Width = mbytes * elements
2375     Scale = TypeSize::Scalable(4);
2376     Width = SVEMaxBytesPerVector / 4;
2377     MinOffset = -8;
2378     MaxOffset = 7;
2379     break;
2380   case AArch64::LD1B_D_IMM:
2381   case AArch64::LD1SB_D_IMM:
2382   case AArch64::ST1B_D_IMM:
2383     // A eighth vector worth of data
2384     // Width = mbytes * elements
2385     Scale = TypeSize::Scalable(2);
2386     Width = SVEMaxBytesPerVector / 8;
2387     MinOffset = -8;
2388     MaxOffset = 7;
2389     break;
2390   case AArch64::ST2GOffset:
2391   case AArch64::STZ2GOffset:
2392     Scale = TypeSize::Fixed(16);
2393     Width = 32;
2394     MinOffset = -256;
2395     MaxOffset = 255;
2396     break;
2397   case AArch64::STGPi:
2398     Scale = TypeSize::Fixed(16);
2399     Width = 16;
2400     MinOffset = -64;
2401     MaxOffset = 63;
2402     break;
2403   }
2404 
2405   return true;
2406 }
2407 
2408 // Scaling factor for unscaled load or store.
2409 int AArch64InstrInfo::getMemScale(unsigned Opc) {
2410   switch (Opc) {
2411   default:
2412     llvm_unreachable("Opcode has unknown scale!");
2413   case AArch64::LDRBBui:
2414   case AArch64::LDURBBi:
2415   case AArch64::LDRSBWui:
2416   case AArch64::LDURSBWi:
2417   case AArch64::STRBBui:
2418   case AArch64::STURBBi:
2419     return 1;
2420   case AArch64::LDRHHui:
2421   case AArch64::LDURHHi:
2422   case AArch64::LDRSHWui:
2423   case AArch64::LDURSHWi:
2424   case AArch64::STRHHui:
2425   case AArch64::STURHHi:
2426     return 2;
2427   case AArch64::LDRSui:
2428   case AArch64::LDURSi:
2429   case AArch64::LDRSWui:
2430   case AArch64::LDURSWi:
2431   case AArch64::LDRWui:
2432   case AArch64::LDURWi:
2433   case AArch64::STRSui:
2434   case AArch64::STURSi:
2435   case AArch64::STRWui:
2436   case AArch64::STURWi:
2437   case AArch64::LDPSi:
2438   case AArch64::LDPSWi:
2439   case AArch64::LDPWi:
2440   case AArch64::STPSi:
2441   case AArch64::STPWi:
2442     return 4;
2443   case AArch64::LDRDui:
2444   case AArch64::LDURDi:
2445   case AArch64::LDRXui:
2446   case AArch64::LDURXi:
2447   case AArch64::STRDui:
2448   case AArch64::STURDi:
2449   case AArch64::STRXui:
2450   case AArch64::STURXi:
2451   case AArch64::LDPDi:
2452   case AArch64::LDPXi:
2453   case AArch64::STPDi:
2454   case AArch64::STPXi:
2455     return 8;
2456   case AArch64::LDRQui:
2457   case AArch64::LDURQi:
2458   case AArch64::STRQui:
2459   case AArch64::STURQi:
2460   case AArch64::LDPQi:
2461   case AArch64::STPQi:
2462   case AArch64::STGOffset:
2463   case AArch64::STZGOffset:
2464   case AArch64::ST2GOffset:
2465   case AArch64::STZ2GOffset:
2466   case AArch64::STGPi:
2467     return 16;
2468   }
2469 }
2470 
2471 // Scale the unscaled offsets.  Returns false if the unscaled offset can't be
2472 // scaled.
2473 static bool scaleOffset(unsigned Opc, int64_t &Offset) {
2474   int Scale = AArch64InstrInfo::getMemScale(Opc);
2475 
2476   // If the byte-offset isn't a multiple of the stride, we can't scale this
2477   // offset.
2478   if (Offset % Scale != 0)
2479     return false;
2480 
2481   // Convert the byte-offset used by unscaled into an "element" offset used
2482   // by the scaled pair load/store instructions.
2483   Offset /= Scale;
2484   return true;
2485 }
2486 
2487 static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
2488   if (FirstOpc == SecondOpc)
2489     return true;
2490   // We can also pair sign-ext and zero-ext instructions.
2491   switch (FirstOpc) {
2492   default:
2493     return false;
2494   case AArch64::LDRWui:
2495   case AArch64::LDURWi:
2496     return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
2497   case AArch64::LDRSWui:
2498   case AArch64::LDURSWi:
2499     return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
2500   }
2501   // These instructions can't be paired based on their opcodes.
2502   return false;
2503 }
2504 
2505 static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
2506                             int64_t Offset1, unsigned Opcode1, int FI2,
2507                             int64_t Offset2, unsigned Opcode2) {
2508   // Accesses through fixed stack object frame indices may access a different
2509   // fixed stack slot. Check that the object offsets + offsets match.
2510   if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
2511     int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
2512     int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
2513     assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
2514     // Convert to scaled object offsets.
2515     int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
2516     if (ObjectOffset1 % Scale1 != 0)
2517       return false;
2518     ObjectOffset1 /= Scale1;
2519     int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
2520     if (ObjectOffset2 % Scale2 != 0)
2521       return false;
2522     ObjectOffset2 /= Scale2;
2523     ObjectOffset1 += Offset1;
2524     ObjectOffset2 += Offset2;
2525     return ObjectOffset1 + 1 == ObjectOffset2;
2526   }
2527 
2528   return FI1 == FI2;
2529 }
2530 
2531 /// Detect opportunities for ldp/stp formation.
2532 ///
2533 /// Only called for LdSt for which getMemOperandWithOffset returns true.
2534 bool AArch64InstrInfo::shouldClusterMemOps(
2535     ArrayRef<const MachineOperand *> BaseOps1,
2536     ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2537     unsigned NumBytes) const {
2538   assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2539   const MachineOperand &BaseOp1 = *BaseOps1.front();
2540   const MachineOperand &BaseOp2 = *BaseOps2.front();
2541   const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2542   const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2543   if (BaseOp1.getType() != BaseOp2.getType())
2544     return false;
2545 
2546   assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2547          "Only base registers and frame indices are supported.");
2548 
2549   // Check for both base regs and base FI.
2550   if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
2551     return false;
2552 
2553   // Only cluster up to a single pair.
2554   if (NumLoads > 2)
2555     return false;
2556 
2557   if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
2558     return false;
2559 
2560   // Can we pair these instructions based on their opcodes?
2561   unsigned FirstOpc = FirstLdSt.getOpcode();
2562   unsigned SecondOpc = SecondLdSt.getOpcode();
2563   if (!canPairLdStOpc(FirstOpc, SecondOpc))
2564     return false;
2565 
2566   // Can't merge volatiles or load/stores that have a hint to avoid pair
2567   // formation, for example.
2568   if (!isCandidateToMergeOrPair(FirstLdSt) ||
2569       !isCandidateToMergeOrPair(SecondLdSt))
2570     return false;
2571 
2572   // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
2573   int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
2574   if (isUnscaledLdSt(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
2575     return false;
2576 
2577   int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
2578   if (isUnscaledLdSt(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
2579     return false;
2580 
2581   // Pairwise instructions have a 7-bit signed offset field.
2582   if (Offset1 > 63 || Offset1 < -64)
2583     return false;
2584 
2585   // The caller should already have ordered First/SecondLdSt by offset.
2586   // Note: except for non-equal frame index bases
2587   if (BaseOp1.isFI()) {
2588     assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
2589            "Caller should have ordered offsets.");
2590 
2591     const MachineFrameInfo &MFI =
2592         FirstLdSt.getParent()->getParent()->getFrameInfo();
2593     return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
2594                            BaseOp2.getIndex(), Offset2, SecondOpc);
2595   }
2596 
2597   assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2598 
2599   return Offset1 + 1 == Offset2;
2600 }
2601 
2602 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
2603                                             unsigned Reg, unsigned SubIdx,
2604                                             unsigned State,
2605                                             const TargetRegisterInfo *TRI) {
2606   if (!SubIdx)
2607     return MIB.addReg(Reg, State);
2608 
2609   if (Register::isPhysicalRegister(Reg))
2610     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
2611   return MIB.addReg(Reg, State, SubIdx);
2612 }
2613 
2614 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
2615                                         unsigned NumRegs) {
2616   // We really want the positive remainder mod 32 here, that happens to be
2617   // easily obtainable with a mask.
2618   return ((DestReg - SrcReg) & 0x1f) < NumRegs;
2619 }
2620 
2621 void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
2622                                         MachineBasicBlock::iterator I,
2623                                         const DebugLoc &DL, MCRegister DestReg,
2624                                         MCRegister SrcReg, bool KillSrc,
2625                                         unsigned Opcode,
2626                                         ArrayRef<unsigned> Indices) const {
2627   assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
2628   const TargetRegisterInfo *TRI = &getRegisterInfo();
2629   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2630   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2631   unsigned NumRegs = Indices.size();
2632 
2633   int SubReg = 0, End = NumRegs, Incr = 1;
2634   if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
2635     SubReg = NumRegs - 1;
2636     End = -1;
2637     Incr = -1;
2638   }
2639 
2640   for (; SubReg != End; SubReg += Incr) {
2641     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2642     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2643     AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
2644     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2645   }
2646 }
2647 
2648 void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
2649                                        MachineBasicBlock::iterator I,
2650                                        DebugLoc DL, unsigned DestReg,
2651                                        unsigned SrcReg, bool KillSrc,
2652                                        unsigned Opcode, unsigned ZeroReg,
2653                                        llvm::ArrayRef<unsigned> Indices) const {
2654   const TargetRegisterInfo *TRI = &getRegisterInfo();
2655   unsigned NumRegs = Indices.size();
2656 
2657 #ifndef NDEBUG
2658   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2659   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2660   assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
2661          "GPR reg sequences should not be able to overlap");
2662 #endif
2663 
2664   for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
2665     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2666     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2667     MIB.addReg(ZeroReg);
2668     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2669     MIB.addImm(0);
2670   }
2671 }
2672 
2673 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
2674                                    MachineBasicBlock::iterator I,
2675                                    const DebugLoc &DL, MCRegister DestReg,
2676                                    MCRegister SrcReg, bool KillSrc) const {
2677   if (AArch64::GPR32spRegClass.contains(DestReg) &&
2678       (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
2679     const TargetRegisterInfo *TRI = &getRegisterInfo();
2680 
2681     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
2682       // If either operand is WSP, expand to ADD #0.
2683       if (Subtarget.hasZeroCycleRegMove()) {
2684         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
2685         MCRegister DestRegX = TRI->getMatchingSuperReg(
2686             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2687         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2688             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2689         // This instruction is reading and writing X registers.  This may upset
2690         // the register scavenger and machine verifier, so we need to indicate
2691         // that we are reading an undefined value from SrcRegX, but a proper
2692         // value from SrcReg.
2693         BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
2694             .addReg(SrcRegX, RegState::Undef)
2695             .addImm(0)
2696             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2697             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2698       } else {
2699         BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
2700             .addReg(SrcReg, getKillRegState(KillSrc))
2701             .addImm(0)
2702             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2703       }
2704     } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
2705       BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
2706           .addImm(0)
2707           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2708     } else {
2709       if (Subtarget.hasZeroCycleRegMove()) {
2710         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
2711         MCRegister DestRegX = TRI->getMatchingSuperReg(
2712             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2713         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2714             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2715         // This instruction is reading and writing X registers.  This may upset
2716         // the register scavenger and machine verifier, so we need to indicate
2717         // that we are reading an undefined value from SrcRegX, but a proper
2718         // value from SrcReg.
2719         BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
2720             .addReg(AArch64::XZR)
2721             .addReg(SrcRegX, RegState::Undef)
2722             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2723       } else {
2724         // Otherwise, expand to ORR WZR.
2725         BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
2726             .addReg(AArch64::WZR)
2727             .addReg(SrcReg, getKillRegState(KillSrc));
2728       }
2729     }
2730     return;
2731   }
2732 
2733   // Copy a Predicate register by ORRing with itself.
2734   if (AArch64::PPRRegClass.contains(DestReg) &&
2735       AArch64::PPRRegClass.contains(SrcReg)) {
2736     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2737     BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
2738       .addReg(SrcReg) // Pg
2739       .addReg(SrcReg)
2740       .addReg(SrcReg, getKillRegState(KillSrc));
2741     return;
2742   }
2743 
2744   // Copy a Z register by ORRing with itself.
2745   if (AArch64::ZPRRegClass.contains(DestReg) &&
2746       AArch64::ZPRRegClass.contains(SrcReg)) {
2747     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2748     BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
2749       .addReg(SrcReg)
2750       .addReg(SrcReg, getKillRegState(KillSrc));
2751     return;
2752   }
2753 
2754   // Copy a Z register pair by copying the individual sub-registers.
2755   if (AArch64::ZPR2RegClass.contains(DestReg) &&
2756       AArch64::ZPR2RegClass.contains(SrcReg)) {
2757     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
2758     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2759                      Indices);
2760     return;
2761   }
2762 
2763   // Copy a Z register triple by copying the individual sub-registers.
2764   if (AArch64::ZPR3RegClass.contains(DestReg) &&
2765       AArch64::ZPR3RegClass.contains(SrcReg)) {
2766     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
2767                                        AArch64::zsub2};
2768     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2769                      Indices);
2770     return;
2771   }
2772 
2773   // Copy a Z register quad by copying the individual sub-registers.
2774   if (AArch64::ZPR4RegClass.contains(DestReg) &&
2775       AArch64::ZPR4RegClass.contains(SrcReg)) {
2776     static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
2777                                        AArch64::zsub2, AArch64::zsub3};
2778     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
2779                      Indices);
2780     return;
2781   }
2782 
2783   if (AArch64::GPR64spRegClass.contains(DestReg) &&
2784       (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
2785     if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
2786       // If either operand is SP, expand to ADD #0.
2787       BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
2788           .addReg(SrcReg, getKillRegState(KillSrc))
2789           .addImm(0)
2790           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2791     } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
2792       BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
2793           .addImm(0)
2794           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2795     } else {
2796       // Otherwise, expand to ORR XZR.
2797       BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
2798           .addReg(AArch64::XZR)
2799           .addReg(SrcReg, getKillRegState(KillSrc));
2800     }
2801     return;
2802   }
2803 
2804   // Copy a DDDD register quad by copying the individual sub-registers.
2805   if (AArch64::DDDDRegClass.contains(DestReg) &&
2806       AArch64::DDDDRegClass.contains(SrcReg)) {
2807     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2808                                        AArch64::dsub2, AArch64::dsub3};
2809     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2810                      Indices);
2811     return;
2812   }
2813 
2814   // Copy a DDD register triple by copying the individual sub-registers.
2815   if (AArch64::DDDRegClass.contains(DestReg) &&
2816       AArch64::DDDRegClass.contains(SrcReg)) {
2817     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2818                                        AArch64::dsub2};
2819     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2820                      Indices);
2821     return;
2822   }
2823 
2824   // Copy a DD register pair by copying the individual sub-registers.
2825   if (AArch64::DDRegClass.contains(DestReg) &&
2826       AArch64::DDRegClass.contains(SrcReg)) {
2827     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
2828     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2829                      Indices);
2830     return;
2831   }
2832 
2833   // Copy a QQQQ register quad by copying the individual sub-registers.
2834   if (AArch64::QQQQRegClass.contains(DestReg) &&
2835       AArch64::QQQQRegClass.contains(SrcReg)) {
2836     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2837                                        AArch64::qsub2, AArch64::qsub3};
2838     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2839                      Indices);
2840     return;
2841   }
2842 
2843   // Copy a QQQ register triple by copying the individual sub-registers.
2844   if (AArch64::QQQRegClass.contains(DestReg) &&
2845       AArch64::QQQRegClass.contains(SrcReg)) {
2846     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2847                                        AArch64::qsub2};
2848     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2849                      Indices);
2850     return;
2851   }
2852 
2853   // Copy a QQ register pair by copying the individual sub-registers.
2854   if (AArch64::QQRegClass.contains(DestReg) &&
2855       AArch64::QQRegClass.contains(SrcReg)) {
2856     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
2857     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2858                      Indices);
2859     return;
2860   }
2861 
2862   if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
2863       AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
2864     static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
2865     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
2866                     AArch64::XZR, Indices);
2867     return;
2868   }
2869 
2870   if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
2871       AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
2872     static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
2873     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
2874                     AArch64::WZR, Indices);
2875     return;
2876   }
2877 
2878   if (AArch64::FPR128RegClass.contains(DestReg) &&
2879       AArch64::FPR128RegClass.contains(SrcReg)) {
2880     if (Subtarget.hasNEON()) {
2881       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2882           .addReg(SrcReg)
2883           .addReg(SrcReg, getKillRegState(KillSrc));
2884     } else {
2885       BuildMI(MBB, I, DL, get(AArch64::STRQpre))
2886           .addReg(AArch64::SP, RegState::Define)
2887           .addReg(SrcReg, getKillRegState(KillSrc))
2888           .addReg(AArch64::SP)
2889           .addImm(-16);
2890       BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
2891           .addReg(AArch64::SP, RegState::Define)
2892           .addReg(DestReg, RegState::Define)
2893           .addReg(AArch64::SP)
2894           .addImm(16);
2895     }
2896     return;
2897   }
2898 
2899   if (AArch64::FPR64RegClass.contains(DestReg) &&
2900       AArch64::FPR64RegClass.contains(SrcReg)) {
2901     if (Subtarget.hasNEON()) {
2902       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
2903                                        &AArch64::FPR128RegClass);
2904       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
2905                                       &AArch64::FPR128RegClass);
2906       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2907           .addReg(SrcReg)
2908           .addReg(SrcReg, getKillRegState(KillSrc));
2909     } else {
2910       BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
2911           .addReg(SrcReg, getKillRegState(KillSrc));
2912     }
2913     return;
2914   }
2915 
2916   if (AArch64::FPR32RegClass.contains(DestReg) &&
2917       AArch64::FPR32RegClass.contains(SrcReg)) {
2918     if (Subtarget.hasNEON()) {
2919       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
2920                                        &AArch64::FPR128RegClass);
2921       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
2922                                       &AArch64::FPR128RegClass);
2923       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2924           .addReg(SrcReg)
2925           .addReg(SrcReg, getKillRegState(KillSrc));
2926     } else {
2927       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2928           .addReg(SrcReg, getKillRegState(KillSrc));
2929     }
2930     return;
2931   }
2932 
2933   if (AArch64::FPR16RegClass.contains(DestReg) &&
2934       AArch64::FPR16RegClass.contains(SrcReg)) {
2935     if (Subtarget.hasNEON()) {
2936       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2937                                        &AArch64::FPR128RegClass);
2938       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2939                                       &AArch64::FPR128RegClass);
2940       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2941           .addReg(SrcReg)
2942           .addReg(SrcReg, getKillRegState(KillSrc));
2943     } else {
2944       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2945                                        &AArch64::FPR32RegClass);
2946       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2947                                       &AArch64::FPR32RegClass);
2948       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2949           .addReg(SrcReg, getKillRegState(KillSrc));
2950     }
2951     return;
2952   }
2953 
2954   if (AArch64::FPR8RegClass.contains(DestReg) &&
2955       AArch64::FPR8RegClass.contains(SrcReg)) {
2956     if (Subtarget.hasNEON()) {
2957       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2958                                        &AArch64::FPR128RegClass);
2959       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2960                                       &AArch64::FPR128RegClass);
2961       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2962           .addReg(SrcReg)
2963           .addReg(SrcReg, getKillRegState(KillSrc));
2964     } else {
2965       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2966                                        &AArch64::FPR32RegClass);
2967       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2968                                       &AArch64::FPR32RegClass);
2969       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2970           .addReg(SrcReg, getKillRegState(KillSrc));
2971     }
2972     return;
2973   }
2974 
2975   // Copies between GPR64 and FPR64.
2976   if (AArch64::FPR64RegClass.contains(DestReg) &&
2977       AArch64::GPR64RegClass.contains(SrcReg)) {
2978     BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
2979         .addReg(SrcReg, getKillRegState(KillSrc));
2980     return;
2981   }
2982   if (AArch64::GPR64RegClass.contains(DestReg) &&
2983       AArch64::FPR64RegClass.contains(SrcReg)) {
2984     BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
2985         .addReg(SrcReg, getKillRegState(KillSrc));
2986     return;
2987   }
2988   // Copies between GPR32 and FPR32.
2989   if (AArch64::FPR32RegClass.contains(DestReg) &&
2990       AArch64::GPR32RegClass.contains(SrcReg)) {
2991     BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
2992         .addReg(SrcReg, getKillRegState(KillSrc));
2993     return;
2994   }
2995   if (AArch64::GPR32RegClass.contains(DestReg) &&
2996       AArch64::FPR32RegClass.contains(SrcReg)) {
2997     BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
2998         .addReg(SrcReg, getKillRegState(KillSrc));
2999     return;
3000   }
3001 
3002   if (DestReg == AArch64::NZCV) {
3003     assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
3004     BuildMI(MBB, I, DL, get(AArch64::MSR))
3005         .addImm(AArch64SysReg::NZCV)
3006         .addReg(SrcReg, getKillRegState(KillSrc))
3007         .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
3008     return;
3009   }
3010 
3011   if (SrcReg == AArch64::NZCV) {
3012     assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
3013     BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
3014         .addImm(AArch64SysReg::NZCV)
3015         .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
3016     return;
3017   }
3018 
3019   llvm_unreachable("unimplemented reg-to-reg copy");
3020 }
3021 
3022 static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
3023                                     MachineBasicBlock &MBB,
3024                                     MachineBasicBlock::iterator InsertBefore,
3025                                     const MCInstrDesc &MCID,
3026                                     Register SrcReg, bool IsKill,
3027                                     unsigned SubIdx0, unsigned SubIdx1, int FI,
3028                                     MachineMemOperand *MMO) {
3029   Register SrcReg0 = SrcReg;
3030   Register SrcReg1 = SrcReg;
3031   if (Register::isPhysicalRegister(SrcReg)) {
3032     SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
3033     SubIdx0 = 0;
3034     SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
3035     SubIdx1 = 0;
3036   }
3037   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3038       .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
3039       .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
3040       .addFrameIndex(FI)
3041       .addImm(0)
3042       .addMemOperand(MMO);
3043 }
3044 
3045 void AArch64InstrInfo::storeRegToStackSlot(
3046     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
3047     bool isKill, int FI, const TargetRegisterClass *RC,
3048     const TargetRegisterInfo *TRI) const {
3049   MachineFunction &MF = *MBB.getParent();
3050   MachineFrameInfo &MFI = MF.getFrameInfo();
3051 
3052   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3053   MachineMemOperand *MMO =
3054       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
3055                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3056   unsigned Opc = 0;
3057   bool Offset = true;
3058   unsigned StackID = TargetStackID::Default;
3059   switch (TRI->getSpillSize(*RC)) {
3060   case 1:
3061     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3062       Opc = AArch64::STRBui;
3063     break;
3064   case 2:
3065     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3066       Opc = AArch64::STRHui;
3067     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3068       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3069       Opc = AArch64::STR_PXI;
3070       StackID = TargetStackID::SVEVector;
3071     }
3072     break;
3073   case 4:
3074     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3075       Opc = AArch64::STRWui;
3076       if (Register::isVirtualRegister(SrcReg))
3077         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
3078       else
3079         assert(SrcReg != AArch64::WSP);
3080     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3081       Opc = AArch64::STRSui;
3082     break;
3083   case 8:
3084     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3085       Opc = AArch64::STRXui;
3086       if (Register::isVirtualRegister(SrcReg))
3087         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3088       else
3089         assert(SrcReg != AArch64::SP);
3090     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3091       Opc = AArch64::STRDui;
3092     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3093       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3094                               get(AArch64::STPWi), SrcReg, isKill,
3095                               AArch64::sube32, AArch64::subo32, FI, MMO);
3096       return;
3097     }
3098     break;
3099   case 16:
3100     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3101       Opc = AArch64::STRQui;
3102     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3103       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3104       Opc = AArch64::ST1Twov1d;
3105       Offset = false;
3106     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3107       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3108                               get(AArch64::STPXi), SrcReg, isKill,
3109                               AArch64::sube64, AArch64::subo64, FI, MMO);
3110       return;
3111     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3112       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3113       Opc = AArch64::STR_ZXI;
3114       StackID = TargetStackID::SVEVector;
3115     }
3116     break;
3117   case 24:
3118     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3119       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3120       Opc = AArch64::ST1Threev1d;
3121       Offset = false;
3122     }
3123     break;
3124   case 32:
3125     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3126       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3127       Opc = AArch64::ST1Fourv1d;
3128       Offset = false;
3129     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3130       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3131       Opc = AArch64::ST1Twov2d;
3132       Offset = false;
3133     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3134       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3135       Opc = AArch64::STR_ZZXI;
3136       StackID = TargetStackID::SVEVector;
3137     }
3138     break;
3139   case 48:
3140     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3141       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3142       Opc = AArch64::ST1Threev2d;
3143       Offset = false;
3144     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3145       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3146       Opc = AArch64::STR_ZZZXI;
3147       StackID = TargetStackID::SVEVector;
3148     }
3149     break;
3150   case 64:
3151     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3152       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3153       Opc = AArch64::ST1Fourv2d;
3154       Offset = false;
3155     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3156       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3157       Opc = AArch64::STR_ZZZZXI;
3158       StackID = TargetStackID::SVEVector;
3159     }
3160     break;
3161   }
3162   assert(Opc && "Unknown register class");
3163   MFI.setStackID(FI, StackID);
3164 
3165   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3166                                      .addReg(SrcReg, getKillRegState(isKill))
3167                                      .addFrameIndex(FI);
3168 
3169   if (Offset)
3170     MI.addImm(0);
3171   MI.addMemOperand(MMO);
3172 }
3173 
3174 static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
3175                                      MachineBasicBlock &MBB,
3176                                      MachineBasicBlock::iterator InsertBefore,
3177                                      const MCInstrDesc &MCID,
3178                                      Register DestReg, unsigned SubIdx0,
3179                                      unsigned SubIdx1, int FI,
3180                                      MachineMemOperand *MMO) {
3181   Register DestReg0 = DestReg;
3182   Register DestReg1 = DestReg;
3183   bool IsUndef = true;
3184   if (Register::isPhysicalRegister(DestReg)) {
3185     DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
3186     SubIdx0 = 0;
3187     DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
3188     SubIdx1 = 0;
3189     IsUndef = false;
3190   }
3191   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3192       .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
3193       .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
3194       .addFrameIndex(FI)
3195       .addImm(0)
3196       .addMemOperand(MMO);
3197 }
3198 
3199 void AArch64InstrInfo::loadRegFromStackSlot(
3200     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
3201     int FI, const TargetRegisterClass *RC,
3202     const TargetRegisterInfo *TRI) const {
3203   MachineFunction &MF = *MBB.getParent();
3204   MachineFrameInfo &MFI = MF.getFrameInfo();
3205   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3206   MachineMemOperand *MMO =
3207       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
3208                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3209 
3210   unsigned Opc = 0;
3211   bool Offset = true;
3212   unsigned StackID = TargetStackID::Default;
3213   switch (TRI->getSpillSize(*RC)) {
3214   case 1:
3215     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3216       Opc = AArch64::LDRBui;
3217     break;
3218   case 2:
3219     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3220       Opc = AArch64::LDRHui;
3221     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3222       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3223       Opc = AArch64::LDR_PXI;
3224       StackID = TargetStackID::SVEVector;
3225     }
3226     break;
3227   case 4:
3228     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3229       Opc = AArch64::LDRWui;
3230       if (Register::isVirtualRegister(DestReg))
3231         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
3232       else
3233         assert(DestReg != AArch64::WSP);
3234     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3235       Opc = AArch64::LDRSui;
3236     break;
3237   case 8:
3238     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3239       Opc = AArch64::LDRXui;
3240       if (Register::isVirtualRegister(DestReg))
3241         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
3242       else
3243         assert(DestReg != AArch64::SP);
3244     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3245       Opc = AArch64::LDRDui;
3246     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3247       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3248                                get(AArch64::LDPWi), DestReg, AArch64::sube32,
3249                                AArch64::subo32, FI, MMO);
3250       return;
3251     }
3252     break;
3253   case 16:
3254     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3255       Opc = AArch64::LDRQui;
3256     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3257       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3258       Opc = AArch64::LD1Twov1d;
3259       Offset = false;
3260     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3261       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3262                                get(AArch64::LDPXi), DestReg, AArch64::sube64,
3263                                AArch64::subo64, FI, MMO);
3264       return;
3265     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3266       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3267       Opc = AArch64::LDR_ZXI;
3268       StackID = TargetStackID::SVEVector;
3269     }
3270     break;
3271   case 24:
3272     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3273       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3274       Opc = AArch64::LD1Threev1d;
3275       Offset = false;
3276     }
3277     break;
3278   case 32:
3279     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3280       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3281       Opc = AArch64::LD1Fourv1d;
3282       Offset = false;
3283     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3284       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3285       Opc = AArch64::LD1Twov2d;
3286       Offset = false;
3287     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3288       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3289       Opc = AArch64::LDR_ZZXI;
3290       StackID = TargetStackID::SVEVector;
3291     }
3292     break;
3293   case 48:
3294     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3295       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3296       Opc = AArch64::LD1Threev2d;
3297       Offset = false;
3298     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3299       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3300       Opc = AArch64::LDR_ZZZXI;
3301       StackID = TargetStackID::SVEVector;
3302     }
3303     break;
3304   case 64:
3305     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3306       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3307       Opc = AArch64::LD1Fourv2d;
3308       Offset = false;
3309     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3310       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3311       Opc = AArch64::LDR_ZZZZXI;
3312       StackID = TargetStackID::SVEVector;
3313     }
3314     break;
3315   }
3316 
3317   assert(Opc && "Unknown register class");
3318   MFI.setStackID(FI, StackID);
3319 
3320   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3321                                      .addReg(DestReg, getDefRegState(true))
3322                                      .addFrameIndex(FI);
3323   if (Offset)
3324     MI.addImm(0);
3325   MI.addMemOperand(MMO);
3326 }
3327 
3328 bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
3329                                            const MachineInstr &UseMI,
3330                                            const TargetRegisterInfo *TRI) {
3331   return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
3332                                          UseMI.getIterator()),
3333                 [TRI](const MachineInstr &I) {
3334                   return I.modifiesRegister(AArch64::NZCV, TRI) ||
3335                          I.readsRegister(AArch64::NZCV, TRI);
3336                 });
3337 }
3338 
3339 // Helper function to emit a frame offset adjustment from a given
3340 // pointer (SrcReg), stored into DestReg. This function is explicit
3341 // in that it requires the opcode.
3342 static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
3343                                MachineBasicBlock::iterator MBBI,
3344                                const DebugLoc &DL, unsigned DestReg,
3345                                unsigned SrcReg, int64_t Offset, unsigned Opc,
3346                                const TargetInstrInfo *TII,
3347                                MachineInstr::MIFlag Flag, bool NeedsWinCFI,
3348                                bool *HasWinCFI) {
3349   int Sign = 1;
3350   unsigned MaxEncoding, ShiftSize;
3351   switch (Opc) {
3352   case AArch64::ADDXri:
3353   case AArch64::ADDSXri:
3354   case AArch64::SUBXri:
3355   case AArch64::SUBSXri:
3356     MaxEncoding = 0xfff;
3357     ShiftSize = 12;
3358     break;
3359   case AArch64::ADDVL_XXI:
3360   case AArch64::ADDPL_XXI:
3361     MaxEncoding = 31;
3362     ShiftSize = 0;
3363     if (Offset < 0) {
3364       MaxEncoding = 32;
3365       Sign = -1;
3366       Offset = -Offset;
3367     }
3368     break;
3369   default:
3370     llvm_unreachable("Unsupported opcode");
3371   }
3372 
3373   // FIXME: If the offset won't fit in 24-bits, compute the offset into a
3374   // scratch register.  If DestReg is a virtual register, use it as the
3375   // scratch register; otherwise, create a new virtual register (to be
3376   // replaced by the scavenger at the end of PEI).  That case can be optimized
3377   // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
3378   // register can be loaded with offset%8 and the add/sub can use an extending
3379   // instruction with LSL#3.
3380   // Currently the function handles any offsets but generates a poor sequence
3381   // of code.
3382   //  assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
3383 
3384   const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
3385   Register TmpReg = DestReg;
3386   if (TmpReg == AArch64::XZR)
3387     TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
3388         &AArch64::GPR64RegClass);
3389   do {
3390     uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
3391     unsigned LocalShiftSize = 0;
3392     if (ThisVal > MaxEncoding) {
3393       ThisVal = ThisVal >> ShiftSize;
3394       LocalShiftSize = ShiftSize;
3395     }
3396     assert((ThisVal >> ShiftSize) <= MaxEncoding &&
3397            "Encoding cannot handle value that big");
3398 
3399     Offset -= ThisVal << LocalShiftSize;
3400     if (Offset == 0)
3401       TmpReg = DestReg;
3402     auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg)
3403                    .addReg(SrcReg)
3404                    .addImm(Sign * (int)ThisVal);
3405     if (ShiftSize)
3406       MBI = MBI.addImm(
3407           AArch64_AM::getShifterImm(AArch64_AM::LSL, LocalShiftSize));
3408     MBI = MBI.setMIFlag(Flag);
3409 
3410     if (NeedsWinCFI) {
3411       assert(Sign == 1 && "SEH directives should always have a positive sign");
3412       int Imm = (int)(ThisVal << LocalShiftSize);
3413       if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
3414           (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
3415         if (HasWinCFI)
3416           *HasWinCFI = true;
3417         if (Imm == 0)
3418           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
3419         else
3420           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
3421               .addImm(Imm)
3422               .setMIFlag(Flag);
3423         assert(Offset == 0 && "Expected remaining offset to be zero to "
3424                               "emit a single SEH directive");
3425       } else if (DestReg == AArch64::SP) {
3426         if (HasWinCFI)
3427           *HasWinCFI = true;
3428         assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
3429         BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
3430             .addImm(Imm)
3431             .setMIFlag(Flag);
3432       }
3433       if (HasWinCFI)
3434         *HasWinCFI = true;
3435     }
3436 
3437     SrcReg = TmpReg;
3438   } while (Offset);
3439 }
3440 
3441 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
3442                            MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
3443                            unsigned DestReg, unsigned SrcReg,
3444                            StackOffset Offset, const TargetInstrInfo *TII,
3445                            MachineInstr::MIFlag Flag, bool SetNZCV,
3446                            bool NeedsWinCFI, bool *HasWinCFI) {
3447   int64_t Bytes, NumPredicateVectors, NumDataVectors;
3448   Offset.getForFrameOffset(Bytes, NumPredicateVectors, NumDataVectors);
3449 
3450   // First emit non-scalable frame offsets, or a simple 'mov'.
3451   if (Bytes || (!Offset && SrcReg != DestReg)) {
3452     assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
3453            "SP increment/decrement not 8-byte aligned");
3454     unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
3455     if (Bytes < 0) {
3456       Bytes = -Bytes;
3457       Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
3458     }
3459     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
3460                        NeedsWinCFI, HasWinCFI);
3461     SrcReg = DestReg;
3462   }
3463 
3464   assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
3465          "SetNZCV not supported with SVE vectors");
3466   assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
3467          "WinCFI not supported with SVE vectors");
3468 
3469   if (NumDataVectors) {
3470     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
3471                        AArch64::ADDVL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3472     SrcReg = DestReg;
3473   }
3474 
3475   if (NumPredicateVectors) {
3476     assert(DestReg != AArch64::SP && "Unaligned access to SP");
3477     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
3478                        AArch64::ADDPL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3479   }
3480 }
3481 
3482 MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
3483     MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
3484     MachineBasicBlock::iterator InsertPt, int FrameIndex,
3485     LiveIntervals *LIS, VirtRegMap *VRM) const {
3486   // This is a bit of a hack. Consider this instruction:
3487   //
3488   //   %0 = COPY %sp; GPR64all:%0
3489   //
3490   // We explicitly chose GPR64all for the virtual register so such a copy might
3491   // be eliminated by RegisterCoalescer. However, that may not be possible, and
3492   // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
3493   // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
3494   //
3495   // To prevent that, we are going to constrain the %0 register class here.
3496   //
3497   // <rdar://problem/11522048>
3498   //
3499   if (MI.isFullCopy()) {
3500     Register DstReg = MI.getOperand(0).getReg();
3501     Register SrcReg = MI.getOperand(1).getReg();
3502     if (SrcReg == AArch64::SP && Register::isVirtualRegister(DstReg)) {
3503       MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
3504       return nullptr;
3505     }
3506     if (DstReg == AArch64::SP && Register::isVirtualRegister(SrcReg)) {
3507       MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3508       return nullptr;
3509     }
3510   }
3511 
3512   // Handle the case where a copy is being spilled or filled but the source
3513   // and destination register class don't match.  For example:
3514   //
3515   //   %0 = COPY %xzr; GPR64common:%0
3516   //
3517   // In this case we can still safely fold away the COPY and generate the
3518   // following spill code:
3519   //
3520   //   STRXui %xzr, %stack.0
3521   //
3522   // This also eliminates spilled cross register class COPYs (e.g. between x and
3523   // d regs) of the same size.  For example:
3524   //
3525   //   %0 = COPY %1; GPR64:%0, FPR64:%1
3526   //
3527   // will be filled as
3528   //
3529   //   LDRDui %0, fi<#0>
3530   //
3531   // instead of
3532   //
3533   //   LDRXui %Temp, fi<#0>
3534   //   %0 = FMOV %Temp
3535   //
3536   if (MI.isCopy() && Ops.size() == 1 &&
3537       // Make sure we're only folding the explicit COPY defs/uses.
3538       (Ops[0] == 0 || Ops[0] == 1)) {
3539     bool IsSpill = Ops[0] == 0;
3540     bool IsFill = !IsSpill;
3541     const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
3542     const MachineRegisterInfo &MRI = MF.getRegInfo();
3543     MachineBasicBlock &MBB = *MI.getParent();
3544     const MachineOperand &DstMO = MI.getOperand(0);
3545     const MachineOperand &SrcMO = MI.getOperand(1);
3546     Register DstReg = DstMO.getReg();
3547     Register SrcReg = SrcMO.getReg();
3548     // This is slightly expensive to compute for physical regs since
3549     // getMinimalPhysRegClass is slow.
3550     auto getRegClass = [&](unsigned Reg) {
3551       return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
3552                                               : TRI.getMinimalPhysRegClass(Reg);
3553     };
3554 
3555     if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
3556       assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
3557                  TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
3558              "Mismatched register size in non subreg COPY");
3559       if (IsSpill)
3560         storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
3561                             getRegClass(SrcReg), &TRI);
3562       else
3563         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
3564                              getRegClass(DstReg), &TRI);
3565       return &*--InsertPt;
3566     }
3567 
3568     // Handle cases like spilling def of:
3569     //
3570     //   %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
3571     //
3572     // where the physical register source can be widened and stored to the full
3573     // virtual reg destination stack slot, in this case producing:
3574     //
3575     //   STRXui %xzr, %stack.0
3576     //
3577     if (IsSpill && DstMO.isUndef() && Register::isPhysicalRegister(SrcReg)) {
3578       assert(SrcMO.getSubReg() == 0 &&
3579              "Unexpected subreg on physical register");
3580       const TargetRegisterClass *SpillRC;
3581       unsigned SpillSubreg;
3582       switch (DstMO.getSubReg()) {
3583       default:
3584         SpillRC = nullptr;
3585         break;
3586       case AArch64::sub_32:
3587       case AArch64::ssub:
3588         if (AArch64::GPR32RegClass.contains(SrcReg)) {
3589           SpillRC = &AArch64::GPR64RegClass;
3590           SpillSubreg = AArch64::sub_32;
3591         } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
3592           SpillRC = &AArch64::FPR64RegClass;
3593           SpillSubreg = AArch64::ssub;
3594         } else
3595           SpillRC = nullptr;
3596         break;
3597       case AArch64::dsub:
3598         if (AArch64::FPR64RegClass.contains(SrcReg)) {
3599           SpillRC = &AArch64::FPR128RegClass;
3600           SpillSubreg = AArch64::dsub;
3601         } else
3602           SpillRC = nullptr;
3603         break;
3604       }
3605 
3606       if (SpillRC)
3607         if (unsigned WidenedSrcReg =
3608                 TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
3609           storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
3610                               FrameIndex, SpillRC, &TRI);
3611           return &*--InsertPt;
3612         }
3613     }
3614 
3615     // Handle cases like filling use of:
3616     //
3617     //   %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
3618     //
3619     // where we can load the full virtual reg source stack slot, into the subreg
3620     // destination, in this case producing:
3621     //
3622     //   LDRWui %0:sub_32<def,read-undef>, %stack.0
3623     //
3624     if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
3625       const TargetRegisterClass *FillRC;
3626       switch (DstMO.getSubReg()) {
3627       default:
3628         FillRC = nullptr;
3629         break;
3630       case AArch64::sub_32:
3631         FillRC = &AArch64::GPR32RegClass;
3632         break;
3633       case AArch64::ssub:
3634         FillRC = &AArch64::FPR32RegClass;
3635         break;
3636       case AArch64::dsub:
3637         FillRC = &AArch64::FPR64RegClass;
3638         break;
3639       }
3640 
3641       if (FillRC) {
3642         assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
3643                    TRI.getRegSizeInBits(*FillRC) &&
3644                "Mismatched regclass size on folded subreg COPY");
3645         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC, &TRI);
3646         MachineInstr &LoadMI = *--InsertPt;
3647         MachineOperand &LoadDst = LoadMI.getOperand(0);
3648         assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
3649         LoadDst.setSubReg(DstMO.getSubReg());
3650         LoadDst.setIsUndef();
3651         return &LoadMI;
3652       }
3653     }
3654   }
3655 
3656   // Cannot fold.
3657   return nullptr;
3658 }
3659 
3660 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
3661                                     StackOffset &SOffset,
3662                                     bool *OutUseUnscaledOp,
3663                                     unsigned *OutUnscaledOp,
3664                                     int64_t *EmittableOffset) {
3665   // Set output values in case of early exit.
3666   if (EmittableOffset)
3667     *EmittableOffset = 0;
3668   if (OutUseUnscaledOp)
3669     *OutUseUnscaledOp = false;
3670   if (OutUnscaledOp)
3671     *OutUnscaledOp = 0;
3672 
3673   // Exit early for structured vector spills/fills as they can't take an
3674   // immediate offset.
3675   switch (MI.getOpcode()) {
3676   default:
3677     break;
3678   case AArch64::LD1Twov2d:
3679   case AArch64::LD1Threev2d:
3680   case AArch64::LD1Fourv2d:
3681   case AArch64::LD1Twov1d:
3682   case AArch64::LD1Threev1d:
3683   case AArch64::LD1Fourv1d:
3684   case AArch64::ST1Twov2d:
3685   case AArch64::ST1Threev2d:
3686   case AArch64::ST1Fourv2d:
3687   case AArch64::ST1Twov1d:
3688   case AArch64::ST1Threev1d:
3689   case AArch64::ST1Fourv1d:
3690   case AArch64::IRG:
3691   case AArch64::IRGstack:
3692   case AArch64::STGloop:
3693   case AArch64::STZGloop:
3694     return AArch64FrameOffsetCannotUpdate;
3695   }
3696 
3697   // Get the min/max offset and the scale.
3698   TypeSize ScaleValue(0U, false);
3699   unsigned Width;
3700   int64_t MinOff, MaxOff;
3701   if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
3702                                       MaxOff))
3703     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3704 
3705   // Construct the complete offset.
3706   bool IsMulVL = ScaleValue.isScalable();
3707   unsigned Scale = ScaleValue.getKnownMinSize();
3708   int64_t Offset = IsMulVL ? SOffset.getScalableBytes() : SOffset.getBytes();
3709 
3710   const MachineOperand &ImmOpnd =
3711       MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
3712   Offset += ImmOpnd.getImm() * Scale;
3713 
3714   // If the offset doesn't match the scale, we rewrite the instruction to
3715   // use the unscaled instruction instead. Likewise, if we have a negative
3716   // offset and there is an unscaled op to use.
3717   Optional<unsigned> UnscaledOp =
3718       AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
3719   bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
3720   if (useUnscaledOp &&
3721       !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
3722                                       MaxOff))
3723     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3724 
3725   Scale = ScaleValue.getKnownMinSize();
3726   assert(IsMulVL == ScaleValue.isScalable() &&
3727          "Unscaled opcode has different value for scalable");
3728 
3729   int64_t Remainder = Offset % Scale;
3730   assert(!(Remainder && useUnscaledOp) &&
3731          "Cannot have remainder when using unscaled op");
3732 
3733   assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
3734   int64_t NewOffset = Offset / Scale;
3735   if (MinOff <= NewOffset && NewOffset <= MaxOff)
3736     Offset = Remainder;
3737   else {
3738     NewOffset = NewOffset < 0 ? MinOff : MaxOff;
3739     Offset = Offset - NewOffset * Scale + Remainder;
3740   }
3741 
3742   if (EmittableOffset)
3743     *EmittableOffset = NewOffset;
3744   if (OutUseUnscaledOp)
3745     *OutUseUnscaledOp = useUnscaledOp;
3746   if (OutUnscaledOp && UnscaledOp)
3747     *OutUnscaledOp = *UnscaledOp;
3748 
3749   if (IsMulVL)
3750     SOffset = StackOffset(Offset, MVT::nxv1i8) +
3751               StackOffset(SOffset.getBytes(), MVT::i8);
3752   else
3753     SOffset = StackOffset(Offset, MVT::i8) +
3754               StackOffset(SOffset.getScalableBytes(), MVT::nxv1i8);
3755   return AArch64FrameOffsetCanUpdate |
3756          (SOffset ? 0 : AArch64FrameOffsetIsLegal);
3757 }
3758 
3759 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
3760                                     unsigned FrameReg, StackOffset &Offset,
3761                                     const AArch64InstrInfo *TII) {
3762   unsigned Opcode = MI.getOpcode();
3763   unsigned ImmIdx = FrameRegIdx + 1;
3764 
3765   if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
3766     Offset += StackOffset(MI.getOperand(ImmIdx).getImm(), MVT::i8);
3767     emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
3768                     MI.getOperand(0).getReg(), FrameReg, Offset, TII,
3769                     MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
3770     MI.eraseFromParent();
3771     Offset = StackOffset();
3772     return true;
3773   }
3774 
3775   int64_t NewOffset;
3776   unsigned UnscaledOp;
3777   bool UseUnscaledOp;
3778   int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
3779                                          &UnscaledOp, &NewOffset);
3780   if (Status & AArch64FrameOffsetCanUpdate) {
3781     if (Status & AArch64FrameOffsetIsLegal)
3782       // Replace the FrameIndex with FrameReg.
3783       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
3784     if (UseUnscaledOp)
3785       MI.setDesc(TII->get(UnscaledOp));
3786 
3787     MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
3788     return !Offset;
3789   }
3790 
3791   return false;
3792 }
3793 
3794 void AArch64InstrInfo::getNoop(MCInst &NopInst) const {
3795   NopInst.setOpcode(AArch64::HINT);
3796   NopInst.addOperand(MCOperand::createImm(0));
3797 }
3798 
3799 // AArch64 supports MachineCombiner.
3800 bool AArch64InstrInfo::useMachineCombiner() const { return true; }
3801 
3802 // True when Opc sets flag
3803 static bool isCombineInstrSettingFlag(unsigned Opc) {
3804   switch (Opc) {
3805   case AArch64::ADDSWrr:
3806   case AArch64::ADDSWri:
3807   case AArch64::ADDSXrr:
3808   case AArch64::ADDSXri:
3809   case AArch64::SUBSWrr:
3810   case AArch64::SUBSXrr:
3811   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3812   case AArch64::SUBSWri:
3813   case AArch64::SUBSXri:
3814     return true;
3815   default:
3816     break;
3817   }
3818   return false;
3819 }
3820 
3821 // 32b Opcodes that can be combined with a MUL
3822 static bool isCombineInstrCandidate32(unsigned Opc) {
3823   switch (Opc) {
3824   case AArch64::ADDWrr:
3825   case AArch64::ADDWri:
3826   case AArch64::SUBWrr:
3827   case AArch64::ADDSWrr:
3828   case AArch64::ADDSWri:
3829   case AArch64::SUBSWrr:
3830   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3831   case AArch64::SUBWri:
3832   case AArch64::SUBSWri:
3833     return true;
3834   default:
3835     break;
3836   }
3837   return false;
3838 }
3839 
3840 // 64b Opcodes that can be combined with a MUL
3841 static bool isCombineInstrCandidate64(unsigned Opc) {
3842   switch (Opc) {
3843   case AArch64::ADDXrr:
3844   case AArch64::ADDXri:
3845   case AArch64::SUBXrr:
3846   case AArch64::ADDSXrr:
3847   case AArch64::ADDSXri:
3848   case AArch64::SUBSXrr:
3849   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3850   case AArch64::SUBXri:
3851   case AArch64::SUBSXri:
3852   case AArch64::ADDv8i8:
3853   case AArch64::ADDv16i8:
3854   case AArch64::ADDv4i16:
3855   case AArch64::ADDv8i16:
3856   case AArch64::ADDv2i32:
3857   case AArch64::ADDv4i32:
3858   case AArch64::SUBv8i8:
3859   case AArch64::SUBv16i8:
3860   case AArch64::SUBv4i16:
3861   case AArch64::SUBv8i16:
3862   case AArch64::SUBv2i32:
3863   case AArch64::SUBv4i32:
3864     return true;
3865   default:
3866     break;
3867   }
3868   return false;
3869 }
3870 
3871 // FP Opcodes that can be combined with a FMUL.
3872 static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
3873   switch (Inst.getOpcode()) {
3874   default:
3875     break;
3876   case AArch64::FADDHrr:
3877   case AArch64::FADDSrr:
3878   case AArch64::FADDDrr:
3879   case AArch64::FADDv4f16:
3880   case AArch64::FADDv8f16:
3881   case AArch64::FADDv2f32:
3882   case AArch64::FADDv2f64:
3883   case AArch64::FADDv4f32:
3884   case AArch64::FSUBHrr:
3885   case AArch64::FSUBSrr:
3886   case AArch64::FSUBDrr:
3887   case AArch64::FSUBv4f16:
3888   case AArch64::FSUBv8f16:
3889   case AArch64::FSUBv2f32:
3890   case AArch64::FSUBv2f64:
3891   case AArch64::FSUBv4f32:
3892     TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
3893     // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
3894     // the target options or if FADD/FSUB has the contract fast-math flag.
3895     return Options.UnsafeFPMath ||
3896            Options.AllowFPOpFusion == FPOpFusion::Fast ||
3897            Inst.getFlag(MachineInstr::FmContract);
3898     return true;
3899   }
3900   return false;
3901 }
3902 
3903 // Opcodes that can be combined with a MUL
3904 static bool isCombineInstrCandidate(unsigned Opc) {
3905   return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
3906 }
3907 
3908 //
3909 // Utility routine that checks if \param MO is defined by an
3910 // \param CombineOpc instruction in the basic block \param MBB
3911 static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
3912                        unsigned CombineOpc, unsigned ZeroReg = 0,
3913                        bool CheckZeroReg = false) {
3914   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3915   MachineInstr *MI = nullptr;
3916 
3917   if (MO.isReg() && Register::isVirtualRegister(MO.getReg()))
3918     MI = MRI.getUniqueVRegDef(MO.getReg());
3919   // And it needs to be in the trace (otherwise, it won't have a depth).
3920   if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc)
3921     return false;
3922   // Must only used by the user we combine with.
3923   if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
3924     return false;
3925 
3926   if (CheckZeroReg) {
3927     assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
3928            MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
3929            MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
3930     // The third input reg must be zero.
3931     if (MI->getOperand(3).getReg() != ZeroReg)
3932       return false;
3933   }
3934 
3935   return true;
3936 }
3937 
3938 //
3939 // Is \param MO defined by an integer multiply and can be combined?
3940 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3941                               unsigned MulOpc, unsigned ZeroReg) {
3942   return canCombine(MBB, MO, MulOpc, ZeroReg, true);
3943 }
3944 
3945 //
3946 // Is \param MO defined by a floating-point multiply and can be combined?
3947 static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3948                                unsigned MulOpc) {
3949   return canCombine(MBB, MO, MulOpc);
3950 }
3951 
3952 // TODO: There are many more machine instruction opcodes to match:
3953 //       1. Other data types (integer, vectors)
3954 //       2. Other math / logic operations (xor, or)
3955 //       3. Other forms of the same operation (intrinsics and other variants)
3956 bool AArch64InstrInfo::isAssociativeAndCommutative(
3957     const MachineInstr &Inst) const {
3958   switch (Inst.getOpcode()) {
3959   case AArch64::FADDDrr:
3960   case AArch64::FADDSrr:
3961   case AArch64::FADDv2f32:
3962   case AArch64::FADDv2f64:
3963   case AArch64::FADDv4f32:
3964   case AArch64::FMULDrr:
3965   case AArch64::FMULSrr:
3966   case AArch64::FMULX32:
3967   case AArch64::FMULX64:
3968   case AArch64::FMULXv2f32:
3969   case AArch64::FMULXv2f64:
3970   case AArch64::FMULXv4f32:
3971   case AArch64::FMULv2f32:
3972   case AArch64::FMULv2f64:
3973   case AArch64::FMULv4f32:
3974     return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath;
3975   default:
3976     return false;
3977   }
3978 }
3979 
3980 /// Find instructions that can be turned into madd.
3981 static bool getMaddPatterns(MachineInstr &Root,
3982                             SmallVectorImpl<MachineCombinerPattern> &Patterns) {
3983   unsigned Opc = Root.getOpcode();
3984   MachineBasicBlock &MBB = *Root.getParent();
3985   bool Found = false;
3986 
3987   if (!isCombineInstrCandidate(Opc))
3988     return false;
3989   if (isCombineInstrSettingFlag(Opc)) {
3990     int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
3991     // When NZCV is live bail out.
3992     if (Cmp_NZCV == -1)
3993       return false;
3994     unsigned NewOpc = convertToNonFlagSettingOpc(Root);
3995     // When opcode can't change bail out.
3996     // CHECKME: do we miss any cases for opcode conversion?
3997     if (NewOpc == Opc)
3998       return false;
3999     Opc = NewOpc;
4000   }
4001 
4002   auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
4003                       MachineCombinerPattern Pattern) {
4004     if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
4005       Patterns.push_back(Pattern);
4006       Found = true;
4007     }
4008   };
4009 
4010   auto setVFound = [&](int Opcode, int Operand, MachineCombinerPattern Pattern) {
4011     if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
4012       Patterns.push_back(Pattern);
4013       Found = true;
4014     }
4015   };
4016 
4017   typedef MachineCombinerPattern MCP;
4018 
4019   switch (Opc) {
4020   default:
4021     break;
4022   case AArch64::ADDWrr:
4023     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4024            "ADDWrr does not have register operands");
4025     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
4026     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
4027     break;
4028   case AArch64::ADDXrr:
4029     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
4030     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
4031     break;
4032   case AArch64::SUBWrr:
4033     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
4034     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
4035     break;
4036   case AArch64::SUBXrr:
4037     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
4038     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
4039     break;
4040   case AArch64::ADDWri:
4041     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
4042     break;
4043   case AArch64::ADDXri:
4044     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
4045     break;
4046   case AArch64::SUBWri:
4047     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
4048     break;
4049   case AArch64::SUBXri:
4050     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
4051     break;
4052   case AArch64::ADDv8i8:
4053     setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
4054     setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
4055     break;
4056   case AArch64::ADDv16i8:
4057     setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
4058     setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
4059     break;
4060   case AArch64::ADDv4i16:
4061     setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
4062     setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
4063     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
4064     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
4065     break;
4066   case AArch64::ADDv8i16:
4067     setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
4068     setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
4069     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
4070     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
4071     break;
4072   case AArch64::ADDv2i32:
4073     setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
4074     setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
4075     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
4076     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
4077     break;
4078   case AArch64::ADDv4i32:
4079     setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
4080     setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
4081     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
4082     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
4083     break;
4084   case AArch64::SUBv8i8:
4085     setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
4086     setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
4087     break;
4088   case AArch64::SUBv16i8:
4089     setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
4090     setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
4091     break;
4092   case AArch64::SUBv4i16:
4093     setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
4094     setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
4095     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
4096     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
4097     break;
4098   case AArch64::SUBv8i16:
4099     setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
4100     setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
4101     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
4102     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
4103     break;
4104   case AArch64::SUBv2i32:
4105     setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
4106     setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
4107     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
4108     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
4109     break;
4110   case AArch64::SUBv4i32:
4111     setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
4112     setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
4113     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
4114     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
4115     break;
4116   }
4117   return Found;
4118 }
4119 /// Floating-Point Support
4120 
4121 /// Find instructions that can be turned into madd.
4122 static bool getFMAPatterns(MachineInstr &Root,
4123                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {
4124 
4125   if (!isCombineInstrCandidateFP(Root))
4126     return false;
4127 
4128   MachineBasicBlock &MBB = *Root.getParent();
4129   bool Found = false;
4130 
4131   auto Match = [&](int Opcode, int Operand,
4132                    MachineCombinerPattern Pattern) -> bool {
4133     if (canCombineWithFMUL(MBB, Root.getOperand(Operand), Opcode)) {
4134       Patterns.push_back(Pattern);
4135       return true;
4136     }
4137     return false;
4138   };
4139 
4140   typedef MachineCombinerPattern MCP;
4141 
4142   switch (Root.getOpcode()) {
4143   default:
4144     assert(false && "Unsupported FP instruction in combiner\n");
4145     break;
4146   case AArch64::FADDHrr:
4147     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4148            "FADDHrr does not have register operands");
4149 
4150     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
4151     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
4152     break;
4153   case AArch64::FADDSrr:
4154     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4155            "FADDSrr does not have register operands");
4156 
4157     Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
4158              Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
4159 
4160     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
4161              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
4162     break;
4163   case AArch64::FADDDrr:
4164     Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
4165              Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
4166 
4167     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
4168              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
4169     break;
4170   case AArch64::FADDv4f16:
4171     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
4172              Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
4173 
4174     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
4175              Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
4176     break;
4177   case AArch64::FADDv8f16:
4178     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
4179              Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
4180 
4181     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
4182              Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
4183     break;
4184   case AArch64::FADDv2f32:
4185     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
4186              Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
4187 
4188     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
4189              Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
4190     break;
4191   case AArch64::FADDv2f64:
4192     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
4193              Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
4194 
4195     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
4196              Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
4197     break;
4198   case AArch64::FADDv4f32:
4199     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
4200              Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
4201 
4202     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
4203              Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
4204     break;
4205   case AArch64::FSUBHrr:
4206     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
4207     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
4208     Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
4209     break;
4210   case AArch64::FSUBSrr:
4211     Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
4212 
4213     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
4214              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
4215 
4216     Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
4217     break;
4218   case AArch64::FSUBDrr:
4219     Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
4220 
4221     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
4222              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
4223 
4224     Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
4225     break;
4226   case AArch64::FSUBv4f16:
4227     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
4228              Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
4229 
4230     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
4231              Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
4232     break;
4233   case AArch64::FSUBv8f16:
4234     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
4235              Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
4236 
4237     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
4238              Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
4239     break;
4240   case AArch64::FSUBv2f32:
4241     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
4242              Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
4243 
4244     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
4245              Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
4246     break;
4247   case AArch64::FSUBv2f64:
4248     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
4249              Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
4250 
4251     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
4252              Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
4253     break;
4254   case AArch64::FSUBv4f32:
4255     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
4256              Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
4257 
4258     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
4259              Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
4260     break;
4261   }
4262   return Found;
4263 }
4264 
4265 /// Return true when a code sequence can improve throughput. It
4266 /// should be called only for instructions in loops.
4267 /// \param Pattern - combiner pattern
4268 bool AArch64InstrInfo::isThroughputPattern(
4269     MachineCombinerPattern Pattern) const {
4270   switch (Pattern) {
4271   default:
4272     break;
4273   case MachineCombinerPattern::FMULADDH_OP1:
4274   case MachineCombinerPattern::FMULADDH_OP2:
4275   case MachineCombinerPattern::FMULSUBH_OP1:
4276   case MachineCombinerPattern::FMULSUBH_OP2:
4277   case MachineCombinerPattern::FMULADDS_OP1:
4278   case MachineCombinerPattern::FMULADDS_OP2:
4279   case MachineCombinerPattern::FMULSUBS_OP1:
4280   case MachineCombinerPattern::FMULSUBS_OP2:
4281   case MachineCombinerPattern::FMULADDD_OP1:
4282   case MachineCombinerPattern::FMULADDD_OP2:
4283   case MachineCombinerPattern::FMULSUBD_OP1:
4284   case MachineCombinerPattern::FMULSUBD_OP2:
4285   case MachineCombinerPattern::FNMULSUBH_OP1:
4286   case MachineCombinerPattern::FNMULSUBS_OP1:
4287   case MachineCombinerPattern::FNMULSUBD_OP1:
4288   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
4289   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
4290   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
4291   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
4292   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4293   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4294   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
4295   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
4296   case MachineCombinerPattern::FMLAv4f16_OP2:
4297   case MachineCombinerPattern::FMLAv4f16_OP1:
4298   case MachineCombinerPattern::FMLAv8f16_OP1:
4299   case MachineCombinerPattern::FMLAv8f16_OP2:
4300   case MachineCombinerPattern::FMLAv2f32_OP2:
4301   case MachineCombinerPattern::FMLAv2f32_OP1:
4302   case MachineCombinerPattern::FMLAv2f64_OP1:
4303   case MachineCombinerPattern::FMLAv2f64_OP2:
4304   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
4305   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
4306   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
4307   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
4308   case MachineCombinerPattern::FMLAv4f32_OP1:
4309   case MachineCombinerPattern::FMLAv4f32_OP2:
4310   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
4311   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
4312   case MachineCombinerPattern::FMLSv4i16_indexed_OP1:
4313   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
4314   case MachineCombinerPattern::FMLSv8i16_indexed_OP1:
4315   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
4316   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
4317   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
4318   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
4319   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
4320   case MachineCombinerPattern::FMLSv4f16_OP1:
4321   case MachineCombinerPattern::FMLSv4f16_OP2:
4322   case MachineCombinerPattern::FMLSv8f16_OP1:
4323   case MachineCombinerPattern::FMLSv8f16_OP2:
4324   case MachineCombinerPattern::FMLSv2f32_OP2:
4325   case MachineCombinerPattern::FMLSv2f64_OP2:
4326   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
4327   case MachineCombinerPattern::FMLSv4f32_OP2:
4328   case MachineCombinerPattern::MULADDv8i8_OP1:
4329   case MachineCombinerPattern::MULADDv8i8_OP2:
4330   case MachineCombinerPattern::MULADDv16i8_OP1:
4331   case MachineCombinerPattern::MULADDv16i8_OP2:
4332   case MachineCombinerPattern::MULADDv4i16_OP1:
4333   case MachineCombinerPattern::MULADDv4i16_OP2:
4334   case MachineCombinerPattern::MULADDv8i16_OP1:
4335   case MachineCombinerPattern::MULADDv8i16_OP2:
4336   case MachineCombinerPattern::MULADDv2i32_OP1:
4337   case MachineCombinerPattern::MULADDv2i32_OP2:
4338   case MachineCombinerPattern::MULADDv4i32_OP1:
4339   case MachineCombinerPattern::MULADDv4i32_OP2:
4340   case MachineCombinerPattern::MULSUBv8i8_OP1:
4341   case MachineCombinerPattern::MULSUBv8i8_OP2:
4342   case MachineCombinerPattern::MULSUBv16i8_OP1:
4343   case MachineCombinerPattern::MULSUBv16i8_OP2:
4344   case MachineCombinerPattern::MULSUBv4i16_OP1:
4345   case MachineCombinerPattern::MULSUBv4i16_OP2:
4346   case MachineCombinerPattern::MULSUBv8i16_OP1:
4347   case MachineCombinerPattern::MULSUBv8i16_OP2:
4348   case MachineCombinerPattern::MULSUBv2i32_OP1:
4349   case MachineCombinerPattern::MULSUBv2i32_OP2:
4350   case MachineCombinerPattern::MULSUBv4i32_OP1:
4351   case MachineCombinerPattern::MULSUBv4i32_OP2:
4352   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4353   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4354   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4355   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4356   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4357   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4358   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4359   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4360   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4361   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4362   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4363   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4364   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4365   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4366   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4367   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4368     return true;
4369   } // end switch (Pattern)
4370   return false;
4371 }
4372 /// Return true when there is potentially a faster code sequence for an
4373 /// instruction chain ending in \p Root. All potential patterns are listed in
4374 /// the \p Pattern vector. Pattern should be sorted in priority order since the
4375 /// pattern evaluator stops checking as soon as it finds a faster sequence.
4376 
4377 bool AArch64InstrInfo::getMachineCombinerPatterns(
4378     MachineInstr &Root,
4379     SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
4380   // Integer patterns
4381   if (getMaddPatterns(Root, Patterns))
4382     return true;
4383   // Floating point patterns
4384   if (getFMAPatterns(Root, Patterns))
4385     return true;
4386 
4387   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
4388 }
4389 
4390 enum class FMAInstKind { Default, Indexed, Accumulator };
4391 /// genFusedMultiply - Generate fused multiply instructions.
4392 /// This function supports both integer and floating point instructions.
4393 /// A typical example:
4394 ///  F|MUL I=A,B,0
4395 ///  F|ADD R,I,C
4396 ///  ==> F|MADD R,A,B,C
4397 /// \param MF Containing MachineFunction
4398 /// \param MRI Register information
4399 /// \param TII Target information
4400 /// \param Root is the F|ADD instruction
4401 /// \param [out] InsInstrs is a vector of machine instructions and will
4402 /// contain the generated madd instruction
4403 /// \param IdxMulOpd is index of operand in Root that is the result of
4404 /// the F|MUL. In the example above IdxMulOpd is 1.
4405 /// \param MaddOpc the opcode fo the f|madd instruction
4406 /// \param RC Register class of operands
4407 /// \param kind of fma instruction (addressing mode) to be generated
4408 /// \param ReplacedAddend is the result register from the instruction
4409 /// replacing the non-combined operand, if any.
4410 static MachineInstr *
4411 genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
4412                  const TargetInstrInfo *TII, MachineInstr &Root,
4413                  SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
4414                  unsigned MaddOpc, const TargetRegisterClass *RC,
4415                  FMAInstKind kind = FMAInstKind::Default,
4416                  const Register *ReplacedAddend = nullptr) {
4417   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4418 
4419   unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
4420   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4421   Register ResultReg = Root.getOperand(0).getReg();
4422   Register SrcReg0 = MUL->getOperand(1).getReg();
4423   bool Src0IsKill = MUL->getOperand(1).isKill();
4424   Register SrcReg1 = MUL->getOperand(2).getReg();
4425   bool Src1IsKill = MUL->getOperand(2).isKill();
4426 
4427   unsigned SrcReg2;
4428   bool Src2IsKill;
4429   if (ReplacedAddend) {
4430     // If we just generated a new addend, we must be it's only use.
4431     SrcReg2 = *ReplacedAddend;
4432     Src2IsKill = true;
4433   } else {
4434     SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
4435     Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
4436   }
4437 
4438   if (Register::isVirtualRegister(ResultReg))
4439     MRI.constrainRegClass(ResultReg, RC);
4440   if (Register::isVirtualRegister(SrcReg0))
4441     MRI.constrainRegClass(SrcReg0, RC);
4442   if (Register::isVirtualRegister(SrcReg1))
4443     MRI.constrainRegClass(SrcReg1, RC);
4444   if (Register::isVirtualRegister(SrcReg2))
4445     MRI.constrainRegClass(SrcReg2, RC);
4446 
4447   MachineInstrBuilder MIB;
4448   if (kind == FMAInstKind::Default)
4449     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4450               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4451               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4452               .addReg(SrcReg2, getKillRegState(Src2IsKill));
4453   else if (kind == FMAInstKind::Indexed)
4454     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4455               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4456               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4457               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4458               .addImm(MUL->getOperand(3).getImm());
4459   else if (kind == FMAInstKind::Accumulator)
4460     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4461               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4462               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4463               .addReg(SrcReg1, getKillRegState(Src1IsKill));
4464   else
4465     assert(false && "Invalid FMA instruction kind \n");
4466   // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
4467   InsInstrs.push_back(MIB);
4468   return MUL;
4469 }
4470 
4471 /// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
4472 /// instructions.
4473 ///
4474 /// \see genFusedMultiply
4475 static MachineInstr *genFusedMultiplyAcc(
4476     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4477     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4478     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4479   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4480                           FMAInstKind::Accumulator);
4481 }
4482 
4483 /// genNeg - Helper to generate an intermediate negation of the second operand
4484 /// of Root
4485 static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
4486                        const TargetInstrInfo *TII, MachineInstr &Root,
4487                        SmallVectorImpl<MachineInstr *> &InsInstrs,
4488                        DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
4489                        unsigned MnegOpc, const TargetRegisterClass *RC) {
4490   Register NewVR = MRI.createVirtualRegister(RC);
4491   MachineInstrBuilder MIB =
4492       BuildMI(MF, Root.getDebugLoc(), TII->get(MnegOpc), NewVR)
4493           .add(Root.getOperand(2));
4494   InsInstrs.push_back(MIB);
4495 
4496   assert(InstrIdxForVirtReg.empty());
4497   InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4498 
4499   return NewVR;
4500 }
4501 
4502 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4503 /// instructions with an additional negation of the accumulator
4504 static MachineInstr *genFusedMultiplyAccNeg(
4505     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4506     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4507     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4508     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4509   assert(IdxMulOpd == 1);
4510 
4511   Register NewVR =
4512       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4513   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4514                           FMAInstKind::Accumulator, &NewVR);
4515 }
4516 
4517 /// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
4518 /// instructions.
4519 ///
4520 /// \see genFusedMultiply
4521 static MachineInstr *genFusedMultiplyIdx(
4522     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4523     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4524     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4525   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4526                           FMAInstKind::Indexed);
4527 }
4528 
4529 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4530 /// instructions with an additional negation of the accumulator
4531 static MachineInstr *genFusedMultiplyIdxNeg(
4532     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4533     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4534     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4535     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4536   assert(IdxMulOpd == 1);
4537 
4538   Register NewVR =
4539       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4540 
4541   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4542                           FMAInstKind::Indexed, &NewVR);
4543 }
4544 
4545 /// genMaddR - Generate madd instruction and combine mul and add using
4546 /// an extra virtual register
4547 /// Example - an ADD intermediate needs to be stored in a register:
4548 ///   MUL I=A,B,0
4549 ///   ADD R,I,Imm
4550 ///   ==> ORR  V, ZR, Imm
4551 ///   ==> MADD R,A,B,V
4552 /// \param MF Containing MachineFunction
4553 /// \param MRI Register information
4554 /// \param TII Target information
4555 /// \param Root is the ADD instruction
4556 /// \param [out] InsInstrs is a vector of machine instructions and will
4557 /// contain the generated madd instruction
4558 /// \param IdxMulOpd is index of operand in Root that is the result of
4559 /// the MUL. In the example above IdxMulOpd is 1.
4560 /// \param MaddOpc the opcode fo the madd instruction
4561 /// \param VR is a virtual register that holds the value of an ADD operand
4562 /// (V in the example above).
4563 /// \param RC Register class of operands
4564 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
4565                               const TargetInstrInfo *TII, MachineInstr &Root,
4566                               SmallVectorImpl<MachineInstr *> &InsInstrs,
4567                               unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
4568                               const TargetRegisterClass *RC) {
4569   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4570 
4571   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4572   Register ResultReg = Root.getOperand(0).getReg();
4573   Register SrcReg0 = MUL->getOperand(1).getReg();
4574   bool Src0IsKill = MUL->getOperand(1).isKill();
4575   Register SrcReg1 = MUL->getOperand(2).getReg();
4576   bool Src1IsKill = MUL->getOperand(2).isKill();
4577 
4578   if (Register::isVirtualRegister(ResultReg))
4579     MRI.constrainRegClass(ResultReg, RC);
4580   if (Register::isVirtualRegister(SrcReg0))
4581     MRI.constrainRegClass(SrcReg0, RC);
4582   if (Register::isVirtualRegister(SrcReg1))
4583     MRI.constrainRegClass(SrcReg1, RC);
4584   if (Register::isVirtualRegister(VR))
4585     MRI.constrainRegClass(VR, RC);
4586 
4587   MachineInstrBuilder MIB =
4588       BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4589           .addReg(SrcReg0, getKillRegState(Src0IsKill))
4590           .addReg(SrcReg1, getKillRegState(Src1IsKill))
4591           .addReg(VR);
4592   // Insert the MADD
4593   InsInstrs.push_back(MIB);
4594   return MUL;
4595 }
4596 
4597 /// When getMachineCombinerPatterns() finds potential patterns,
4598 /// this function generates the instructions that could replace the
4599 /// original code sequence
4600 void AArch64InstrInfo::genAlternativeCodeSequence(
4601     MachineInstr &Root, MachineCombinerPattern Pattern,
4602     SmallVectorImpl<MachineInstr *> &InsInstrs,
4603     SmallVectorImpl<MachineInstr *> &DelInstrs,
4604     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
4605   MachineBasicBlock &MBB = *Root.getParent();
4606   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4607   MachineFunction &MF = *MBB.getParent();
4608   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
4609 
4610   MachineInstr *MUL;
4611   const TargetRegisterClass *RC;
4612   unsigned Opc;
4613   switch (Pattern) {
4614   default:
4615     // Reassociate instructions.
4616     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
4617                                                 DelInstrs, InstrIdxForVirtReg);
4618     return;
4619   case MachineCombinerPattern::MULADDW_OP1:
4620   case MachineCombinerPattern::MULADDX_OP1:
4621     // MUL I=A,B,0
4622     // ADD R,I,C
4623     // ==> MADD R,A,B,C
4624     // --- Create(MADD);
4625     if (Pattern == MachineCombinerPattern::MULADDW_OP1) {
4626       Opc = AArch64::MADDWrrr;
4627       RC = &AArch64::GPR32RegClass;
4628     } else {
4629       Opc = AArch64::MADDXrrr;
4630       RC = &AArch64::GPR64RegClass;
4631     }
4632     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4633     break;
4634   case MachineCombinerPattern::MULADDW_OP2:
4635   case MachineCombinerPattern::MULADDX_OP2:
4636     // MUL I=A,B,0
4637     // ADD R,C,I
4638     // ==> MADD R,A,B,C
4639     // --- Create(MADD);
4640     if (Pattern == MachineCombinerPattern::MULADDW_OP2) {
4641       Opc = AArch64::MADDWrrr;
4642       RC = &AArch64::GPR32RegClass;
4643     } else {
4644       Opc = AArch64::MADDXrrr;
4645       RC = &AArch64::GPR64RegClass;
4646     }
4647     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4648     break;
4649   case MachineCombinerPattern::MULADDWI_OP1:
4650   case MachineCombinerPattern::MULADDXI_OP1: {
4651     // MUL I=A,B,0
4652     // ADD R,I,Imm
4653     // ==> ORR  V, ZR, Imm
4654     // ==> MADD R,A,B,V
4655     // --- Create(MADD);
4656     const TargetRegisterClass *OrrRC;
4657     unsigned BitSize, OrrOpc, ZeroReg;
4658     if (Pattern == MachineCombinerPattern::MULADDWI_OP1) {
4659       OrrOpc = AArch64::ORRWri;
4660       OrrRC = &AArch64::GPR32spRegClass;
4661       BitSize = 32;
4662       ZeroReg = AArch64::WZR;
4663       Opc = AArch64::MADDWrrr;
4664       RC = &AArch64::GPR32RegClass;
4665     } else {
4666       OrrOpc = AArch64::ORRXri;
4667       OrrRC = &AArch64::GPR64spRegClass;
4668       BitSize = 64;
4669       ZeroReg = AArch64::XZR;
4670       Opc = AArch64::MADDXrrr;
4671       RC = &AArch64::GPR64RegClass;
4672     }
4673     Register NewVR = MRI.createVirtualRegister(OrrRC);
4674     uint64_t Imm = Root.getOperand(2).getImm();
4675 
4676     if (Root.getOperand(3).isImm()) {
4677       unsigned Val = Root.getOperand(3).getImm();
4678       Imm = Imm << Val;
4679     }
4680     uint64_t UImm = SignExtend64(Imm, BitSize);
4681     uint64_t Encoding;
4682     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4683       MachineInstrBuilder MIB1 =
4684           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4685               .addReg(ZeroReg)
4686               .addImm(Encoding);
4687       InsInstrs.push_back(MIB1);
4688       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4689       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4690     }
4691     break;
4692   }
4693   case MachineCombinerPattern::MULSUBW_OP1:
4694   case MachineCombinerPattern::MULSUBX_OP1: {
4695     // MUL I=A,B,0
4696     // SUB R,I, C
4697     // ==> SUB  V, 0, C
4698     // ==> MADD R,A,B,V // = -C + A*B
4699     // --- Create(MADD);
4700     const TargetRegisterClass *SubRC;
4701     unsigned SubOpc, ZeroReg;
4702     if (Pattern == MachineCombinerPattern::MULSUBW_OP1) {
4703       SubOpc = AArch64::SUBWrr;
4704       SubRC = &AArch64::GPR32spRegClass;
4705       ZeroReg = AArch64::WZR;
4706       Opc = AArch64::MADDWrrr;
4707       RC = &AArch64::GPR32RegClass;
4708     } else {
4709       SubOpc = AArch64::SUBXrr;
4710       SubRC = &AArch64::GPR64spRegClass;
4711       ZeroReg = AArch64::XZR;
4712       Opc = AArch64::MADDXrrr;
4713       RC = &AArch64::GPR64RegClass;
4714     }
4715     Register NewVR = MRI.createVirtualRegister(SubRC);
4716     // SUB NewVR, 0, C
4717     MachineInstrBuilder MIB1 =
4718         BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
4719             .addReg(ZeroReg)
4720             .add(Root.getOperand(2));
4721     InsInstrs.push_back(MIB1);
4722     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4723     MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4724     break;
4725   }
4726   case MachineCombinerPattern::MULSUBW_OP2:
4727   case MachineCombinerPattern::MULSUBX_OP2:
4728     // MUL I=A,B,0
4729     // SUB R,C,I
4730     // ==> MSUB R,A,B,C (computes C - A*B)
4731     // --- Create(MSUB);
4732     if (Pattern == MachineCombinerPattern::MULSUBW_OP2) {
4733       Opc = AArch64::MSUBWrrr;
4734       RC = &AArch64::GPR32RegClass;
4735     } else {
4736       Opc = AArch64::MSUBXrrr;
4737       RC = &AArch64::GPR64RegClass;
4738     }
4739     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4740     break;
4741   case MachineCombinerPattern::MULSUBWI_OP1:
4742   case MachineCombinerPattern::MULSUBXI_OP1: {
4743     // MUL I=A,B,0
4744     // SUB R,I, Imm
4745     // ==> ORR  V, ZR, -Imm
4746     // ==> MADD R,A,B,V // = -Imm + A*B
4747     // --- Create(MADD);
4748     const TargetRegisterClass *OrrRC;
4749     unsigned BitSize, OrrOpc, ZeroReg;
4750     if (Pattern == MachineCombinerPattern::MULSUBWI_OP1) {
4751       OrrOpc = AArch64::ORRWri;
4752       OrrRC = &AArch64::GPR32spRegClass;
4753       BitSize = 32;
4754       ZeroReg = AArch64::WZR;
4755       Opc = AArch64::MADDWrrr;
4756       RC = &AArch64::GPR32RegClass;
4757     } else {
4758       OrrOpc = AArch64::ORRXri;
4759       OrrRC = &AArch64::GPR64spRegClass;
4760       BitSize = 64;
4761       ZeroReg = AArch64::XZR;
4762       Opc = AArch64::MADDXrrr;
4763       RC = &AArch64::GPR64RegClass;
4764     }
4765     Register NewVR = MRI.createVirtualRegister(OrrRC);
4766     uint64_t Imm = Root.getOperand(2).getImm();
4767     if (Root.getOperand(3).isImm()) {
4768       unsigned Val = Root.getOperand(3).getImm();
4769       Imm = Imm << Val;
4770     }
4771     uint64_t UImm = SignExtend64(-Imm, BitSize);
4772     uint64_t Encoding;
4773     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4774       MachineInstrBuilder MIB1 =
4775           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4776               .addReg(ZeroReg)
4777               .addImm(Encoding);
4778       InsInstrs.push_back(MIB1);
4779       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4780       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4781     }
4782     break;
4783   }
4784 
4785   case MachineCombinerPattern::MULADDv8i8_OP1:
4786     Opc = AArch64::MLAv8i8;
4787     RC = &AArch64::FPR64RegClass;
4788     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4789     break;
4790   case MachineCombinerPattern::MULADDv8i8_OP2:
4791     Opc = AArch64::MLAv8i8;
4792     RC = &AArch64::FPR64RegClass;
4793     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4794     break;
4795   case MachineCombinerPattern::MULADDv16i8_OP1:
4796     Opc = AArch64::MLAv16i8;
4797     RC = &AArch64::FPR128RegClass;
4798     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4799     break;
4800   case MachineCombinerPattern::MULADDv16i8_OP2:
4801     Opc = AArch64::MLAv16i8;
4802     RC = &AArch64::FPR128RegClass;
4803     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4804     break;
4805   case MachineCombinerPattern::MULADDv4i16_OP1:
4806     Opc = AArch64::MLAv4i16;
4807     RC = &AArch64::FPR64RegClass;
4808     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4809     break;
4810   case MachineCombinerPattern::MULADDv4i16_OP2:
4811     Opc = AArch64::MLAv4i16;
4812     RC = &AArch64::FPR64RegClass;
4813     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4814     break;
4815   case MachineCombinerPattern::MULADDv8i16_OP1:
4816     Opc = AArch64::MLAv8i16;
4817     RC = &AArch64::FPR128RegClass;
4818     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4819     break;
4820   case MachineCombinerPattern::MULADDv8i16_OP2:
4821     Opc = AArch64::MLAv8i16;
4822     RC = &AArch64::FPR128RegClass;
4823     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4824     break;
4825   case MachineCombinerPattern::MULADDv2i32_OP1:
4826     Opc = AArch64::MLAv2i32;
4827     RC = &AArch64::FPR64RegClass;
4828     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4829     break;
4830   case MachineCombinerPattern::MULADDv2i32_OP2:
4831     Opc = AArch64::MLAv2i32;
4832     RC = &AArch64::FPR64RegClass;
4833     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4834     break;
4835   case MachineCombinerPattern::MULADDv4i32_OP1:
4836     Opc = AArch64::MLAv4i32;
4837     RC = &AArch64::FPR128RegClass;
4838     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4839     break;
4840   case MachineCombinerPattern::MULADDv4i32_OP2:
4841     Opc = AArch64::MLAv4i32;
4842     RC = &AArch64::FPR128RegClass;
4843     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4844     break;
4845 
4846   case MachineCombinerPattern::MULSUBv8i8_OP1:
4847     Opc = AArch64::MLAv8i8;
4848     RC = &AArch64::FPR64RegClass;
4849     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4850                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
4851                                  RC);
4852     break;
4853   case MachineCombinerPattern::MULSUBv8i8_OP2:
4854     Opc = AArch64::MLSv8i8;
4855     RC = &AArch64::FPR64RegClass;
4856     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4857     break;
4858   case MachineCombinerPattern::MULSUBv16i8_OP1:
4859     Opc = AArch64::MLAv16i8;
4860     RC = &AArch64::FPR128RegClass;
4861     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4862                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
4863                                  RC);
4864     break;
4865   case MachineCombinerPattern::MULSUBv16i8_OP2:
4866     Opc = AArch64::MLSv16i8;
4867     RC = &AArch64::FPR128RegClass;
4868     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4869     break;
4870   case MachineCombinerPattern::MULSUBv4i16_OP1:
4871     Opc = AArch64::MLAv4i16;
4872     RC = &AArch64::FPR64RegClass;
4873     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4874                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4875                                  RC);
4876     break;
4877   case MachineCombinerPattern::MULSUBv4i16_OP2:
4878     Opc = AArch64::MLSv4i16;
4879     RC = &AArch64::FPR64RegClass;
4880     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4881     break;
4882   case MachineCombinerPattern::MULSUBv8i16_OP1:
4883     Opc = AArch64::MLAv8i16;
4884     RC = &AArch64::FPR128RegClass;
4885     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4886                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4887                                  RC);
4888     break;
4889   case MachineCombinerPattern::MULSUBv8i16_OP2:
4890     Opc = AArch64::MLSv8i16;
4891     RC = &AArch64::FPR128RegClass;
4892     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4893     break;
4894   case MachineCombinerPattern::MULSUBv2i32_OP1:
4895     Opc = AArch64::MLAv2i32;
4896     RC = &AArch64::FPR64RegClass;
4897     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4898                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4899                                  RC);
4900     break;
4901   case MachineCombinerPattern::MULSUBv2i32_OP2:
4902     Opc = AArch64::MLSv2i32;
4903     RC = &AArch64::FPR64RegClass;
4904     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4905     break;
4906   case MachineCombinerPattern::MULSUBv4i32_OP1:
4907     Opc = AArch64::MLAv4i32;
4908     RC = &AArch64::FPR128RegClass;
4909     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4910                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
4911                                  RC);
4912     break;
4913   case MachineCombinerPattern::MULSUBv4i32_OP2:
4914     Opc = AArch64::MLSv4i32;
4915     RC = &AArch64::FPR128RegClass;
4916     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4917     break;
4918 
4919   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4920     Opc = AArch64::MLAv4i16_indexed;
4921     RC = &AArch64::FPR64RegClass;
4922     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4923     break;
4924   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4925     Opc = AArch64::MLAv4i16_indexed;
4926     RC = &AArch64::FPR64RegClass;
4927     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4928     break;
4929   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4930     Opc = AArch64::MLAv8i16_indexed;
4931     RC = &AArch64::FPR128RegClass;
4932     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4933     break;
4934   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4935     Opc = AArch64::MLAv8i16_indexed;
4936     RC = &AArch64::FPR128RegClass;
4937     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4938     break;
4939   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4940     Opc = AArch64::MLAv2i32_indexed;
4941     RC = &AArch64::FPR64RegClass;
4942     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4943     break;
4944   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4945     Opc = AArch64::MLAv2i32_indexed;
4946     RC = &AArch64::FPR64RegClass;
4947     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4948     break;
4949   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4950     Opc = AArch64::MLAv4i32_indexed;
4951     RC = &AArch64::FPR128RegClass;
4952     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4953     break;
4954   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4955     Opc = AArch64::MLAv4i32_indexed;
4956     RC = &AArch64::FPR128RegClass;
4957     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4958     break;
4959 
4960   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4961     Opc = AArch64::MLAv4i16_indexed;
4962     RC = &AArch64::FPR64RegClass;
4963     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4964                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4965                                  RC);
4966     break;
4967   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4968     Opc = AArch64::MLSv4i16_indexed;
4969     RC = &AArch64::FPR64RegClass;
4970     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4971     break;
4972   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4973     Opc = AArch64::MLAv8i16_indexed;
4974     RC = &AArch64::FPR128RegClass;
4975     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4976                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4977                                  RC);
4978     break;
4979   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4980     Opc = AArch64::MLSv8i16_indexed;
4981     RC = &AArch64::FPR128RegClass;
4982     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4983     break;
4984   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4985     Opc = AArch64::MLAv2i32_indexed;
4986     RC = &AArch64::FPR64RegClass;
4987     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4988                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4989                                  RC);
4990     break;
4991   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4992     Opc = AArch64::MLSv2i32_indexed;
4993     RC = &AArch64::FPR64RegClass;
4994     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4995     break;
4996   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4997     Opc = AArch64::MLAv4i32_indexed;
4998     RC = &AArch64::FPR128RegClass;
4999     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
5000                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
5001                                  RC);
5002     break;
5003   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
5004     Opc = AArch64::MLSv4i32_indexed;
5005     RC = &AArch64::FPR128RegClass;
5006     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5007     break;
5008 
5009   // Floating Point Support
5010   case MachineCombinerPattern::FMULADDH_OP1:
5011     Opc = AArch64::FMADDHrrr;
5012     RC = &AArch64::FPR16RegClass;
5013     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5014     break;
5015   case MachineCombinerPattern::FMULADDS_OP1:
5016     Opc = AArch64::FMADDSrrr;
5017     RC = &AArch64::FPR32RegClass;
5018     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5019     break;
5020   case MachineCombinerPattern::FMULADDD_OP1:
5021     Opc = AArch64::FMADDDrrr;
5022     RC = &AArch64::FPR64RegClass;
5023     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5024     break;
5025 
5026   case MachineCombinerPattern::FMULADDH_OP2:
5027     Opc = AArch64::FMADDHrrr;
5028     RC = &AArch64::FPR16RegClass;
5029     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5030     break;
5031   case MachineCombinerPattern::FMULADDS_OP2:
5032     Opc = AArch64::FMADDSrrr;
5033     RC = &AArch64::FPR32RegClass;
5034     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5035     break;
5036   case MachineCombinerPattern::FMULADDD_OP2:
5037     Opc = AArch64::FMADDDrrr;
5038     RC = &AArch64::FPR64RegClass;
5039     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5040     break;
5041 
5042   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
5043     Opc = AArch64::FMLAv1i32_indexed;
5044     RC = &AArch64::FPR32RegClass;
5045     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5046                            FMAInstKind::Indexed);
5047     break;
5048   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
5049     Opc = AArch64::FMLAv1i32_indexed;
5050     RC = &AArch64::FPR32RegClass;
5051     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5052                            FMAInstKind::Indexed);
5053     break;
5054 
5055   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
5056     Opc = AArch64::FMLAv1i64_indexed;
5057     RC = &AArch64::FPR64RegClass;
5058     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5059                            FMAInstKind::Indexed);
5060     break;
5061   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
5062     Opc = AArch64::FMLAv1i64_indexed;
5063     RC = &AArch64::FPR64RegClass;
5064     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5065                            FMAInstKind::Indexed);
5066     break;
5067 
5068   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
5069     RC = &AArch64::FPR64RegClass;
5070     Opc = AArch64::FMLAv4i16_indexed;
5071     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5072                            FMAInstKind::Indexed);
5073     break;
5074   case MachineCombinerPattern::FMLAv4f16_OP1:
5075     RC = &AArch64::FPR64RegClass;
5076     Opc = AArch64::FMLAv4f16;
5077     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5078                            FMAInstKind::Accumulator);
5079     break;
5080   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
5081     RC = &AArch64::FPR64RegClass;
5082     Opc = AArch64::FMLAv4i16_indexed;
5083     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5084                            FMAInstKind::Indexed);
5085     break;
5086   case MachineCombinerPattern::FMLAv4f16_OP2:
5087     RC = &AArch64::FPR64RegClass;
5088     Opc = AArch64::FMLAv4f16;
5089     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5090                            FMAInstKind::Accumulator);
5091     break;
5092 
5093   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
5094   case MachineCombinerPattern::FMLAv2f32_OP1:
5095     RC = &AArch64::FPR64RegClass;
5096     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
5097       Opc = AArch64::FMLAv2i32_indexed;
5098       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5099                              FMAInstKind::Indexed);
5100     } else {
5101       Opc = AArch64::FMLAv2f32;
5102       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5103                              FMAInstKind::Accumulator);
5104     }
5105     break;
5106   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
5107   case MachineCombinerPattern::FMLAv2f32_OP2:
5108     RC = &AArch64::FPR64RegClass;
5109     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
5110       Opc = AArch64::FMLAv2i32_indexed;
5111       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5112                              FMAInstKind::Indexed);
5113     } else {
5114       Opc = AArch64::FMLAv2f32;
5115       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5116                              FMAInstKind::Accumulator);
5117     }
5118     break;
5119 
5120   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
5121     RC = &AArch64::FPR128RegClass;
5122     Opc = AArch64::FMLAv8i16_indexed;
5123     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5124                            FMAInstKind::Indexed);
5125     break;
5126   case MachineCombinerPattern::FMLAv8f16_OP1:
5127     RC = &AArch64::FPR128RegClass;
5128     Opc = AArch64::FMLAv8f16;
5129     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5130                            FMAInstKind::Accumulator);
5131     break;
5132   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
5133     RC = &AArch64::FPR128RegClass;
5134     Opc = AArch64::FMLAv8i16_indexed;
5135     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5136                            FMAInstKind::Indexed);
5137     break;
5138   case MachineCombinerPattern::FMLAv8f16_OP2:
5139     RC = &AArch64::FPR128RegClass;
5140     Opc = AArch64::FMLAv8f16;
5141     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5142                            FMAInstKind::Accumulator);
5143     break;
5144 
5145   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
5146   case MachineCombinerPattern::FMLAv2f64_OP1:
5147     RC = &AArch64::FPR128RegClass;
5148     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
5149       Opc = AArch64::FMLAv2i64_indexed;
5150       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5151                              FMAInstKind::Indexed);
5152     } else {
5153       Opc = AArch64::FMLAv2f64;
5154       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5155                              FMAInstKind::Accumulator);
5156     }
5157     break;
5158   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
5159   case MachineCombinerPattern::FMLAv2f64_OP2:
5160     RC = &AArch64::FPR128RegClass;
5161     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
5162       Opc = AArch64::FMLAv2i64_indexed;
5163       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5164                              FMAInstKind::Indexed);
5165     } else {
5166       Opc = AArch64::FMLAv2f64;
5167       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5168                              FMAInstKind::Accumulator);
5169     }
5170     break;
5171 
5172   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
5173   case MachineCombinerPattern::FMLAv4f32_OP1:
5174     RC = &AArch64::FPR128RegClass;
5175     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
5176       Opc = AArch64::FMLAv4i32_indexed;
5177       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5178                              FMAInstKind::Indexed);
5179     } else {
5180       Opc = AArch64::FMLAv4f32;
5181       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5182                              FMAInstKind::Accumulator);
5183     }
5184     break;
5185 
5186   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
5187   case MachineCombinerPattern::FMLAv4f32_OP2:
5188     RC = &AArch64::FPR128RegClass;
5189     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
5190       Opc = AArch64::FMLAv4i32_indexed;
5191       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5192                              FMAInstKind::Indexed);
5193     } else {
5194       Opc = AArch64::FMLAv4f32;
5195       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5196                              FMAInstKind::Accumulator);
5197     }
5198     break;
5199 
5200   case MachineCombinerPattern::FMULSUBH_OP1:
5201     Opc = AArch64::FNMSUBHrrr;
5202     RC = &AArch64::FPR16RegClass;
5203     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5204     break;
5205   case MachineCombinerPattern::FMULSUBS_OP1:
5206     Opc = AArch64::FNMSUBSrrr;
5207     RC = &AArch64::FPR32RegClass;
5208     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5209     break;
5210   case MachineCombinerPattern::FMULSUBD_OP1:
5211     Opc = AArch64::FNMSUBDrrr;
5212     RC = &AArch64::FPR64RegClass;
5213     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5214     break;
5215 
5216   case MachineCombinerPattern::FNMULSUBH_OP1:
5217     Opc = AArch64::FNMADDHrrr;
5218     RC = &AArch64::FPR16RegClass;
5219     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5220     break;
5221   case MachineCombinerPattern::FNMULSUBS_OP1:
5222     Opc = AArch64::FNMADDSrrr;
5223     RC = &AArch64::FPR32RegClass;
5224     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5225     break;
5226   case MachineCombinerPattern::FNMULSUBD_OP1:
5227     Opc = AArch64::FNMADDDrrr;
5228     RC = &AArch64::FPR64RegClass;
5229     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5230     break;
5231 
5232   case MachineCombinerPattern::FMULSUBH_OP2:
5233     Opc = AArch64::FMSUBHrrr;
5234     RC = &AArch64::FPR16RegClass;
5235     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5236     break;
5237   case MachineCombinerPattern::FMULSUBS_OP2:
5238     Opc = AArch64::FMSUBSrrr;
5239     RC = &AArch64::FPR32RegClass;
5240     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5241     break;
5242   case MachineCombinerPattern::FMULSUBD_OP2:
5243     Opc = AArch64::FMSUBDrrr;
5244     RC = &AArch64::FPR64RegClass;
5245     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5246     break;
5247 
5248   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
5249     Opc = AArch64::FMLSv1i32_indexed;
5250     RC = &AArch64::FPR32RegClass;
5251     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5252                            FMAInstKind::Indexed);
5253     break;
5254 
5255   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
5256     Opc = AArch64::FMLSv1i64_indexed;
5257     RC = &AArch64::FPR64RegClass;
5258     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5259                            FMAInstKind::Indexed);
5260     break;
5261 
5262   case MachineCombinerPattern::FMLSv4f16_OP1:
5263   case MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
5264     RC = &AArch64::FPR64RegClass;
5265     Register NewVR = MRI.createVirtualRegister(RC);
5266     MachineInstrBuilder MIB1 =
5267         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f16), NewVR)
5268             .add(Root.getOperand(2));
5269     InsInstrs.push_back(MIB1);
5270     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5271     if (Pattern == MachineCombinerPattern::FMLSv4f16_OP1) {
5272       Opc = AArch64::FMLAv4f16;
5273       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5274                              FMAInstKind::Accumulator, &NewVR);
5275     } else {
5276       Opc = AArch64::FMLAv4i16_indexed;
5277       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5278                              FMAInstKind::Indexed, &NewVR);
5279     }
5280     break;
5281   }
5282   case MachineCombinerPattern::FMLSv4f16_OP2:
5283     RC = &AArch64::FPR64RegClass;
5284     Opc = AArch64::FMLSv4f16;
5285     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5286                            FMAInstKind::Accumulator);
5287     break;
5288   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
5289     RC = &AArch64::FPR64RegClass;
5290     Opc = AArch64::FMLSv4i16_indexed;
5291     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5292                            FMAInstKind::Indexed);
5293     break;
5294 
5295   case MachineCombinerPattern::FMLSv2f32_OP2:
5296   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
5297     RC = &AArch64::FPR64RegClass;
5298     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
5299       Opc = AArch64::FMLSv2i32_indexed;
5300       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5301                              FMAInstKind::Indexed);
5302     } else {
5303       Opc = AArch64::FMLSv2f32;
5304       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5305                              FMAInstKind::Accumulator);
5306     }
5307     break;
5308 
5309   case MachineCombinerPattern::FMLSv8f16_OP1:
5310   case MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
5311     RC = &AArch64::FPR128RegClass;
5312     Register NewVR = MRI.createVirtualRegister(RC);
5313     MachineInstrBuilder MIB1 =
5314         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv8f16), NewVR)
5315             .add(Root.getOperand(2));
5316     InsInstrs.push_back(MIB1);
5317     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5318     if (Pattern == MachineCombinerPattern::FMLSv8f16_OP1) {
5319       Opc = AArch64::FMLAv8f16;
5320       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5321                              FMAInstKind::Accumulator, &NewVR);
5322     } else {
5323       Opc = AArch64::FMLAv8i16_indexed;
5324       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5325                              FMAInstKind::Indexed, &NewVR);
5326     }
5327     break;
5328   }
5329   case MachineCombinerPattern::FMLSv8f16_OP2:
5330     RC = &AArch64::FPR128RegClass;
5331     Opc = AArch64::FMLSv8f16;
5332     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5333                            FMAInstKind::Accumulator);
5334     break;
5335   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
5336     RC = &AArch64::FPR128RegClass;
5337     Opc = AArch64::FMLSv8i16_indexed;
5338     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5339                            FMAInstKind::Indexed);
5340     break;
5341 
5342   case MachineCombinerPattern::FMLSv2f64_OP2:
5343   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
5344     RC = &AArch64::FPR128RegClass;
5345     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
5346       Opc = AArch64::FMLSv2i64_indexed;
5347       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5348                              FMAInstKind::Indexed);
5349     } else {
5350       Opc = AArch64::FMLSv2f64;
5351       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5352                              FMAInstKind::Accumulator);
5353     }
5354     break;
5355 
5356   case MachineCombinerPattern::FMLSv4f32_OP2:
5357   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
5358     RC = &AArch64::FPR128RegClass;
5359     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
5360       Opc = AArch64::FMLSv4i32_indexed;
5361       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5362                              FMAInstKind::Indexed);
5363     } else {
5364       Opc = AArch64::FMLSv4f32;
5365       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5366                              FMAInstKind::Accumulator);
5367     }
5368     break;
5369   case MachineCombinerPattern::FMLSv2f32_OP1:
5370   case MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
5371     RC = &AArch64::FPR64RegClass;
5372     Register NewVR = MRI.createVirtualRegister(RC);
5373     MachineInstrBuilder MIB1 =
5374         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f32), NewVR)
5375             .add(Root.getOperand(2));
5376     InsInstrs.push_back(MIB1);
5377     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5378     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
5379       Opc = AArch64::FMLAv2i32_indexed;
5380       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5381                              FMAInstKind::Indexed, &NewVR);
5382     } else {
5383       Opc = AArch64::FMLAv2f32;
5384       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5385                              FMAInstKind::Accumulator, &NewVR);
5386     }
5387     break;
5388   }
5389   case MachineCombinerPattern::FMLSv4f32_OP1:
5390   case MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
5391     RC = &AArch64::FPR128RegClass;
5392     Register NewVR = MRI.createVirtualRegister(RC);
5393     MachineInstrBuilder MIB1 =
5394         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f32), NewVR)
5395             .add(Root.getOperand(2));
5396     InsInstrs.push_back(MIB1);
5397     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5398     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
5399       Opc = AArch64::FMLAv4i32_indexed;
5400       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5401                              FMAInstKind::Indexed, &NewVR);
5402     } else {
5403       Opc = AArch64::FMLAv4f32;
5404       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5405                              FMAInstKind::Accumulator, &NewVR);
5406     }
5407     break;
5408   }
5409   case MachineCombinerPattern::FMLSv2f64_OP1:
5410   case MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
5411     RC = &AArch64::FPR128RegClass;
5412     Register NewVR = MRI.createVirtualRegister(RC);
5413     MachineInstrBuilder MIB1 =
5414         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f64), NewVR)
5415             .add(Root.getOperand(2));
5416     InsInstrs.push_back(MIB1);
5417     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5418     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
5419       Opc = AArch64::FMLAv2i64_indexed;
5420       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5421                              FMAInstKind::Indexed, &NewVR);
5422     } else {
5423       Opc = AArch64::FMLAv2f64;
5424       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5425                              FMAInstKind::Accumulator, &NewVR);
5426     }
5427     break;
5428   }
5429   } // end switch (Pattern)
5430   // Record MUL and ADD/SUB for deletion
5431   DelInstrs.push_back(MUL);
5432   DelInstrs.push_back(&Root);
5433 }
5434 
5435 /// Replace csincr-branch sequence by simple conditional branch
5436 ///
5437 /// Examples:
5438 /// 1. \code
5439 ///   csinc  w9, wzr, wzr, <condition code>
5440 ///   tbnz   w9, #0, 0x44
5441 ///    \endcode
5442 /// to
5443 ///    \code
5444 ///   b.<inverted condition code>
5445 ///    \endcode
5446 ///
5447 /// 2. \code
5448 ///   csinc w9, wzr, wzr, <condition code>
5449 ///   tbz   w9, #0, 0x44
5450 ///    \endcode
5451 /// to
5452 ///    \code
5453 ///   b.<condition code>
5454 ///    \endcode
5455 ///
5456 /// Replace compare and branch sequence by TBZ/TBNZ instruction when the
5457 /// compare's constant operand is power of 2.
5458 ///
5459 /// Examples:
5460 ///    \code
5461 ///   and  w8, w8, #0x400
5462 ///   cbnz w8, L1
5463 ///    \endcode
5464 /// to
5465 ///    \code
5466 ///   tbnz w8, #10, L1
5467 ///    \endcode
5468 ///
5469 /// \param  MI Conditional Branch
5470 /// \return True when the simple conditional branch is generated
5471 ///
5472 bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
5473   bool IsNegativeBranch = false;
5474   bool IsTestAndBranch = false;
5475   unsigned TargetBBInMI = 0;
5476   switch (MI.getOpcode()) {
5477   default:
5478     llvm_unreachable("Unknown branch instruction?");
5479   case AArch64::Bcc:
5480     return false;
5481   case AArch64::CBZW:
5482   case AArch64::CBZX:
5483     TargetBBInMI = 1;
5484     break;
5485   case AArch64::CBNZW:
5486   case AArch64::CBNZX:
5487     TargetBBInMI = 1;
5488     IsNegativeBranch = true;
5489     break;
5490   case AArch64::TBZW:
5491   case AArch64::TBZX:
5492     TargetBBInMI = 2;
5493     IsTestAndBranch = true;
5494     break;
5495   case AArch64::TBNZW:
5496   case AArch64::TBNZX:
5497     TargetBBInMI = 2;
5498     IsNegativeBranch = true;
5499     IsTestAndBranch = true;
5500     break;
5501   }
5502   // So we increment a zero register and test for bits other
5503   // than bit 0? Conservatively bail out in case the verifier
5504   // missed this case.
5505   if (IsTestAndBranch && MI.getOperand(1).getImm())
5506     return false;
5507 
5508   // Find Definition.
5509   assert(MI.getParent() && "Incomplete machine instruciton\n");
5510   MachineBasicBlock *MBB = MI.getParent();
5511   MachineFunction *MF = MBB->getParent();
5512   MachineRegisterInfo *MRI = &MF->getRegInfo();
5513   Register VReg = MI.getOperand(0).getReg();
5514   if (!Register::isVirtualRegister(VReg))
5515     return false;
5516 
5517   MachineInstr *DefMI = MRI->getVRegDef(VReg);
5518 
5519   // Look through COPY instructions to find definition.
5520   while (DefMI->isCopy()) {
5521     Register CopyVReg = DefMI->getOperand(1).getReg();
5522     if (!MRI->hasOneNonDBGUse(CopyVReg))
5523       return false;
5524     if (!MRI->hasOneDef(CopyVReg))
5525       return false;
5526     DefMI = MRI->getVRegDef(CopyVReg);
5527   }
5528 
5529   switch (DefMI->getOpcode()) {
5530   default:
5531     return false;
5532   // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
5533   case AArch64::ANDWri:
5534   case AArch64::ANDXri: {
5535     if (IsTestAndBranch)
5536       return false;
5537     if (DefMI->getParent() != MBB)
5538       return false;
5539     if (!MRI->hasOneNonDBGUse(VReg))
5540       return false;
5541 
5542     bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
5543     uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
5544         DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
5545     if (!isPowerOf2_64(Mask))
5546       return false;
5547 
5548     MachineOperand &MO = DefMI->getOperand(1);
5549     Register NewReg = MO.getReg();
5550     if (!Register::isVirtualRegister(NewReg))
5551       return false;
5552 
5553     assert(!MRI->def_empty(NewReg) && "Register must be defined.");
5554 
5555     MachineBasicBlock &RefToMBB = *MBB;
5556     MachineBasicBlock *TBB = MI.getOperand(1).getMBB();
5557     DebugLoc DL = MI.getDebugLoc();
5558     unsigned Imm = Log2_64(Mask);
5559     unsigned Opc = (Imm < 32)
5560                        ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
5561                        : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
5562     MachineInstr *NewMI = BuildMI(RefToMBB, MI, DL, get(Opc))
5563                               .addReg(NewReg)
5564                               .addImm(Imm)
5565                               .addMBB(TBB);
5566     // Register lives on to the CBZ now.
5567     MO.setIsKill(false);
5568 
5569     // For immediate smaller than 32, we need to use the 32-bit
5570     // variant (W) in all cases. Indeed the 64-bit variant does not
5571     // allow to encode them.
5572     // Therefore, if the input register is 64-bit, we need to take the
5573     // 32-bit sub-part.
5574     if (!Is32Bit && Imm < 32)
5575       NewMI->getOperand(0).setSubReg(AArch64::sub_32);
5576     MI.eraseFromParent();
5577     return true;
5578   }
5579   // Look for CSINC
5580   case AArch64::CSINCWr:
5581   case AArch64::CSINCXr: {
5582     if (!(DefMI->getOperand(1).getReg() == AArch64::WZR &&
5583           DefMI->getOperand(2).getReg() == AArch64::WZR) &&
5584         !(DefMI->getOperand(1).getReg() == AArch64::XZR &&
5585           DefMI->getOperand(2).getReg() == AArch64::XZR))
5586       return false;
5587 
5588     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
5589       return false;
5590 
5591     AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(3).getImm();
5592     // Convert only when the condition code is not modified between
5593     // the CSINC and the branch. The CC may be used by other
5594     // instructions in between.
5595     if (areCFlagsAccessedBetweenInstrs(DefMI, MI, &getRegisterInfo(), AK_Write))
5596       return false;
5597     MachineBasicBlock &RefToMBB = *MBB;
5598     MachineBasicBlock *TBB = MI.getOperand(TargetBBInMI).getMBB();
5599     DebugLoc DL = MI.getDebugLoc();
5600     if (IsNegativeBranch)
5601       CC = AArch64CC::getInvertedCondCode(CC);
5602     BuildMI(RefToMBB, MI, DL, get(AArch64::Bcc)).addImm(CC).addMBB(TBB);
5603     MI.eraseFromParent();
5604     return true;
5605   }
5606   }
5607 }
5608 
5609 std::pair<unsigned, unsigned>
5610 AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5611   const unsigned Mask = AArch64II::MO_FRAGMENT;
5612   return std::make_pair(TF & Mask, TF & ~Mask);
5613 }
5614 
5615 ArrayRef<std::pair<unsigned, const char *>>
5616 AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5617   using namespace AArch64II;
5618 
5619   static const std::pair<unsigned, const char *> TargetFlags[] = {
5620       {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
5621       {MO_G3, "aarch64-g3"},     {MO_G2, "aarch64-g2"},
5622       {MO_G1, "aarch64-g1"},     {MO_G0, "aarch64-g0"},
5623       {MO_HI12, "aarch64-hi12"}};
5624   return makeArrayRef(TargetFlags);
5625 }
5626 
5627 ArrayRef<std::pair<unsigned, const char *>>
5628 AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5629   using namespace AArch64II;
5630 
5631   static const std::pair<unsigned, const char *> TargetFlags[] = {
5632       {MO_COFFSTUB, "aarch64-coffstub"},
5633       {MO_GOT, "aarch64-got"},
5634       {MO_NC, "aarch64-nc"},
5635       {MO_S, "aarch64-s"},
5636       {MO_TLS, "aarch64-tls"},
5637       {MO_DLLIMPORT, "aarch64-dllimport"},
5638       {MO_PREL, "aarch64-prel"},
5639       {MO_TAGGED, "aarch64-tagged"}};
5640   return makeArrayRef(TargetFlags);
5641 }
5642 
5643 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5644 AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5645   static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5646       {{MOSuppressPair, "aarch64-suppress-pair"},
5647        {MOStridedAccess, "aarch64-strided-access"}};
5648   return makeArrayRef(TargetFlags);
5649 }
5650 
5651 /// Constants defining how certain sequences should be outlined.
5652 /// This encompasses how an outlined function should be called, and what kind of
5653 /// frame should be emitted for that outlined function.
5654 ///
5655 /// \p MachineOutlinerDefault implies that the function should be called with
5656 /// a save and restore of LR to the stack.
5657 ///
5658 /// That is,
5659 ///
5660 /// I1     Save LR                    OUTLINED_FUNCTION:
5661 /// I2 --> BL OUTLINED_FUNCTION       I1
5662 /// I3     Restore LR                 I2
5663 ///                                   I3
5664 ///                                   RET
5665 ///
5666 /// * Call construction overhead: 3 (save + BL + restore)
5667 /// * Frame construction overhead: 1 (ret)
5668 /// * Requires stack fixups? Yes
5669 ///
5670 /// \p MachineOutlinerTailCall implies that the function is being created from
5671 /// a sequence of instructions ending in a return.
5672 ///
5673 /// That is,
5674 ///
5675 /// I1                             OUTLINED_FUNCTION:
5676 /// I2 --> B OUTLINED_FUNCTION     I1
5677 /// RET                            I2
5678 ///                                RET
5679 ///
5680 /// * Call construction overhead: 1 (B)
5681 /// * Frame construction overhead: 0 (Return included in sequence)
5682 /// * Requires stack fixups? No
5683 ///
5684 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5685 /// a BL instruction, but doesn't require LR to be saved and restored. This
5686 /// happens when LR is known to be dead.
5687 ///
5688 /// That is,
5689 ///
5690 /// I1                                OUTLINED_FUNCTION:
5691 /// I2 --> BL OUTLINED_FUNCTION       I1
5692 /// I3                                I2
5693 ///                                   I3
5694 ///                                   RET
5695 ///
5696 /// * Call construction overhead: 1 (BL)
5697 /// * Frame construction overhead: 1 (RET)
5698 /// * Requires stack fixups? No
5699 ///
5700 /// \p MachineOutlinerThunk implies that the function is being created from
5701 /// a sequence of instructions ending in a call. The outlined function is
5702 /// called with a BL instruction, and the outlined function tail-calls the
5703 /// original call destination.
5704 ///
5705 /// That is,
5706 ///
5707 /// I1                                OUTLINED_FUNCTION:
5708 /// I2 --> BL OUTLINED_FUNCTION       I1
5709 /// BL f                              I2
5710 ///                                   B f
5711 /// * Call construction overhead: 1 (BL)
5712 /// * Frame construction overhead: 0
5713 /// * Requires stack fixups? No
5714 ///
5715 /// \p MachineOutlinerRegSave implies that the function should be called with a
5716 /// save and restore of LR to an available register. This allows us to avoid
5717 /// stack fixups. Note that this outlining variant is compatible with the
5718 /// NoLRSave case.
5719 ///
5720 /// That is,
5721 ///
5722 /// I1     Save LR                    OUTLINED_FUNCTION:
5723 /// I2 --> BL OUTLINED_FUNCTION       I1
5724 /// I3     Restore LR                 I2
5725 ///                                   I3
5726 ///                                   RET
5727 ///
5728 /// * Call construction overhead: 3 (save + BL + restore)
5729 /// * Frame construction overhead: 1 (ret)
5730 /// * Requires stack fixups? No
5731 enum MachineOutlinerClass {
5732   MachineOutlinerDefault,  /// Emit a save, restore, call, and return.
5733   MachineOutlinerTailCall, /// Only emit a branch.
5734   MachineOutlinerNoLRSave, /// Emit a call and return.
5735   MachineOutlinerThunk,    /// Emit a call and tail-call.
5736   MachineOutlinerRegSave   /// Same as default, but save to a register.
5737 };
5738 
5739 enum MachineOutlinerMBBFlags {
5740   LRUnavailableSomewhere = 0x2,
5741   HasCalls = 0x4,
5742   UnsafeRegsDead = 0x8
5743 };
5744 
5745 unsigned
5746 AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5747   assert(C.LRUWasSet && "LRU wasn't set?");
5748   MachineFunction *MF = C.getMF();
5749   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
5750       MF->getSubtarget().getRegisterInfo());
5751 
5752   // Check if there is an available register across the sequence that we can
5753   // use.
5754   for (unsigned Reg : AArch64::GPR64RegClass) {
5755     if (!ARI->isReservedReg(*MF, Reg) &&
5756         Reg != AArch64::LR &&  // LR is not reserved, but don't use it.
5757         Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
5758         Reg != AArch64::X17 && // Ditto for X17.
5759         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5760       return Reg;
5761   }
5762 
5763   // No suitable register. Return 0.
5764   return 0u;
5765 }
5766 
5767 static bool
5768 outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
5769                                          const outliner::Candidate &b) {
5770   const Function &Fa = a.getMF()->getFunction();
5771   const Function &Fb = b.getMF()->getFunction();
5772 
5773   // If none of the functions have the "sign-return-address" attribute their
5774   // signing behaviour is equal
5775   if (!Fa.hasFnAttribute("sign-return-address") &&
5776       !Fb.hasFnAttribute("sign-return-address")) {
5777     return true;
5778   }
5779 
5780   // If both functions have the "sign-return-address" attribute their signing
5781   // behaviour is equal, if the values of the attributes are equal
5782   if (Fa.hasFnAttribute("sign-return-address") &&
5783       Fb.hasFnAttribute("sign-return-address")) {
5784     StringRef ScopeA =
5785         Fa.getFnAttribute("sign-return-address").getValueAsString();
5786     StringRef ScopeB =
5787         Fb.getFnAttribute("sign-return-address").getValueAsString();
5788     return ScopeA.equals(ScopeB);
5789   }
5790 
5791   // If function B doesn't have the "sign-return-address" attribute but A does,
5792   // the functions' signing behaviour is equal if A's value for
5793   // "sign-return-address" is "none" and vice versa.
5794   if (Fa.hasFnAttribute("sign-return-address")) {
5795     StringRef ScopeA =
5796         Fa.getFnAttribute("sign-return-address").getValueAsString();
5797     return ScopeA.equals("none");
5798   }
5799 
5800   if (Fb.hasFnAttribute("sign-return-address")) {
5801     StringRef ScopeB =
5802         Fb.getFnAttribute("sign-return-address").getValueAsString();
5803     return ScopeB.equals("none");
5804   }
5805 
5806   llvm_unreachable("Unkown combination of sign-return-address attributes");
5807 }
5808 
5809 static bool
5810 outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
5811                                        const outliner::Candidate &b) {
5812   const Function &Fa = a.getMF()->getFunction();
5813   const Function &Fb = b.getMF()->getFunction();
5814 
5815   // If none of the functions have the "sign-return-address-key" attribute
5816   // their keys are equal
5817   if (!Fa.hasFnAttribute("sign-return-address-key") &&
5818       !Fb.hasFnAttribute("sign-return-address-key")) {
5819     return true;
5820   }
5821 
5822   // If both functions have the "sign-return-address-key" attribute their
5823   // keys are equal if the values of "sign-return-address-key" are equal
5824   if (Fa.hasFnAttribute("sign-return-address-key") &&
5825       Fb.hasFnAttribute("sign-return-address-key")) {
5826     StringRef KeyA =
5827         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5828     StringRef KeyB =
5829         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5830     return KeyA.equals(KeyB);
5831   }
5832 
5833   // If B doesn't have the "sign-return-address-key" attribute, both keys are
5834   // equal, if function a has the default key (a_key)
5835   if (Fa.hasFnAttribute("sign-return-address-key")) {
5836     StringRef KeyA =
5837         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5838     return KeyA.equals_lower("a_key");
5839   }
5840 
5841   if (Fb.hasFnAttribute("sign-return-address-key")) {
5842     StringRef KeyB =
5843         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5844     return KeyB.equals_lower("a_key");
5845   }
5846 
5847   llvm_unreachable("Unkown combination of sign-return-address-key attributes");
5848 }
5849 
5850 static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
5851                                                 const outliner::Candidate &b) {
5852   const AArch64Subtarget &SubtargetA =
5853       a.getMF()->getSubtarget<AArch64Subtarget>();
5854   const AArch64Subtarget &SubtargetB =
5855       b.getMF()->getSubtarget<AArch64Subtarget>();
5856   return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
5857 }
5858 
5859 outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
5860     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5861   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5862   unsigned SequenceSize =
5863       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5864                       [this](unsigned Sum, const MachineInstr &MI) {
5865                         return Sum + getInstSizeInBytes(MI);
5866                       });
5867   unsigned NumBytesToCreateFrame = 0;
5868 
5869   // We only allow outlining for functions having exactly matching return
5870   // address signing attributes, i.e., all share the same value for the
5871   // attribute "sign-return-address" and all share the same type of key they
5872   // are signed with.
5873   // Additionally we require all functions to simultaniously either support
5874   // v8.3a features or not. Otherwise an outlined function could get signed
5875   // using dedicated v8.3 instructions and a call from a function that doesn't
5876   // support v8.3 instructions would therefore be invalid.
5877   if (std::adjacent_find(
5878           RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5879           [](const outliner::Candidate &a, const outliner::Candidate &b) {
5880             // Return true if a and b are non-equal w.r.t. return address
5881             // signing or support of v8.3a features
5882             if (outliningCandidatesSigningScopeConsensus(a, b) &&
5883                 outliningCandidatesSigningKeyConsensus(a, b) &&
5884                 outliningCandidatesV8_3OpsConsensus(a, b)) {
5885               return false;
5886             }
5887             return true;
5888           }) != RepeatedSequenceLocs.end()) {
5889     return outliner::OutlinedFunction();
5890   }
5891 
5892   // Since at this point all candidates agree on their return address signing
5893   // picking just one is fine. If the candidate functions potentially sign their
5894   // return addresses, the outlined function should do the same. Note that in
5895   // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5896   // not certainly true that the outlined function will have to sign its return
5897   // address but this decision is made later, when the decision to outline
5898   // has already been made.
5899   // The same holds for the number of additional instructions we need: On
5900   // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5901   // necessary. However, at this point we don't know if the outlined function
5902   // will have a RET instruction so we assume the worst.
5903   const Function &FCF = FirstCand.getMF()->getFunction();
5904   const TargetRegisterInfo &TRI = getRegisterInfo();
5905   if (FCF.hasFnAttribute("sign-return-address")) {
5906     // One PAC and one AUT instructions
5907     NumBytesToCreateFrame += 8;
5908 
5909     // We have to check if sp modifying instructions would get outlined.
5910     // If so we only allow outlining if sp is unchanged overall, so matching
5911     // sub and add instructions are okay to outline, all other sp modifications
5912     // are not
5913     auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5914       int SPValue = 0;
5915       MachineBasicBlock::iterator MBBI = C.front();
5916       for (;;) {
5917         if (MBBI->modifiesRegister(AArch64::SP, &TRI)) {
5918           switch (MBBI->getOpcode()) {
5919           case AArch64::ADDXri:
5920           case AArch64::ADDWri:
5921             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5922             assert(MBBI->getOperand(2).isImm() &&
5923                    "Expected operand to be immediate");
5924             assert(MBBI->getOperand(1).isReg() &&
5925                    "Expected operand to be a register");
5926             // Check if the add just increments sp. If so, we search for
5927             // matching sub instructions that decrement sp. If not, the
5928             // modification is illegal
5929             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5930               SPValue += MBBI->getOperand(2).getImm();
5931             else
5932               return true;
5933             break;
5934           case AArch64::SUBXri:
5935           case AArch64::SUBWri:
5936             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5937             assert(MBBI->getOperand(2).isImm() &&
5938                    "Expected operand to be immediate");
5939             assert(MBBI->getOperand(1).isReg() &&
5940                    "Expected operand to be a register");
5941             // Check if the sub just decrements sp. If so, we search for
5942             // matching add instructions that increment sp. If not, the
5943             // modification is illegal
5944             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5945               SPValue -= MBBI->getOperand(2).getImm();
5946             else
5947               return true;
5948             break;
5949           default:
5950             return true;
5951           }
5952         }
5953         if (MBBI == C.back())
5954           break;
5955         ++MBBI;
5956       }
5957       if (SPValue)
5958         return true;
5959       return false;
5960     };
5961     // Remove candidates with illegal stack modifying instructions
5962     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5963                                               RepeatedSequenceLocs.end(),
5964                                               hasIllegalSPModification),
5965                                RepeatedSequenceLocs.end());
5966 
5967     // If the sequence doesn't have enough candidates left, then we're done.
5968     if (RepeatedSequenceLocs.size() < 2)
5969       return outliner::OutlinedFunction();
5970   }
5971 
5972   // Properties about candidate MBBs that hold for all of them.
5973   unsigned FlagsSetInAll = 0xF;
5974 
5975   // Compute liveness information for each candidate, and set FlagsSetInAll.
5976   std::for_each(RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5977                 [&FlagsSetInAll](outliner::Candidate &C) {
5978                   FlagsSetInAll &= C.Flags;
5979                 });
5980 
5981   // According to the AArch64 Procedure Call Standard, the following are
5982   // undefined on entry/exit from a function call:
5983   //
5984   // * Registers x16, x17, (and thus w16, w17)
5985   // * Condition codes (and thus the NZCV register)
5986   //
5987   // Because if this, we can't outline any sequence of instructions where
5988   // one
5989   // of these registers is live into/across it. Thus, we need to delete
5990   // those
5991   // candidates.
5992   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5993     // If the unsafe registers in this block are all dead, then we don't need
5994     // to compute liveness here.
5995     if (C.Flags & UnsafeRegsDead)
5996       return false;
5997     C.initLRU(TRI);
5998     LiveRegUnits LRU = C.LRU;
5999     return (!LRU.available(AArch64::W16) || !LRU.available(AArch64::W17) ||
6000             !LRU.available(AArch64::NZCV));
6001   };
6002 
6003   // Are there any candidates where those registers are live?
6004   if (!(FlagsSetInAll & UnsafeRegsDead)) {
6005     // Erase every candidate that violates the restrictions above. (It could be
6006     // true that we have viable candidates, so it's not worth bailing out in
6007     // the case that, say, 1 out of 20 candidates violate the restructions.)
6008     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
6009                                               RepeatedSequenceLocs.end(),
6010                                               CantGuaranteeValueAcrossCall),
6011                                RepeatedSequenceLocs.end());
6012 
6013     // If the sequence doesn't have enough candidates left, then we're done.
6014     if (RepeatedSequenceLocs.size() < 2)
6015       return outliner::OutlinedFunction();
6016   }
6017 
6018   // At this point, we have only "safe" candidates to outline. Figure out
6019   // frame + call instruction information.
6020 
6021   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
6022 
6023   // Helper lambda which sets call information for every candidate.
6024   auto SetCandidateCallInfo =
6025       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
6026         for (outliner::Candidate &C : RepeatedSequenceLocs)
6027           C.setCallInfo(CallID, NumBytesForCall);
6028       };
6029 
6030   unsigned FrameID = MachineOutlinerDefault;
6031   NumBytesToCreateFrame += 4;
6032 
6033   bool HasBTI = any_of(RepeatedSequenceLocs, [](outliner::Candidate &C) {
6034     return C.getMF()->getFunction().hasFnAttribute("branch-target-enforcement");
6035   });
6036 
6037   // We check to see if CFI Instructions are present, and if they are
6038   // we find the number of CFI Instructions in the candidates.
6039   unsigned CFICount = 0;
6040   MachineBasicBlock::iterator MBBI = RepeatedSequenceLocs[0].front();
6041   for (unsigned Loc = RepeatedSequenceLocs[0].getStartIdx();
6042        Loc < RepeatedSequenceLocs[0].getEndIdx() + 1; Loc++) {
6043     const std::vector<MCCFIInstruction> &CFIInstructions =
6044         RepeatedSequenceLocs[0].getMF()->getFrameInstructions();
6045     if (MBBI->isCFIInstruction()) {
6046       unsigned CFIIndex = MBBI->getOperand(0).getCFIIndex();
6047       MCCFIInstruction CFI = CFIInstructions[CFIIndex];
6048       CFICount++;
6049     }
6050     MBBI++;
6051   }
6052 
6053   // We compare the number of found CFI Instructions to  the number of CFI
6054   // instructions in the parent function for each candidate.  We must check this
6055   // since if we outline one of the CFI instructions in a function, we have to
6056   // outline them all for correctness. If we do not, the address offsets will be
6057   // incorrect between the two sections of the program.
6058   for (outliner::Candidate &C : RepeatedSequenceLocs) {
6059     std::vector<MCCFIInstruction> CFIInstructions =
6060         C.getMF()->getFrameInstructions();
6061 
6062     if (CFICount > 0 && CFICount != CFIInstructions.size())
6063       return outliner::OutlinedFunction();
6064   }
6065 
6066   // Returns true if an instructions is safe to fix up, false otherwise.
6067   auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
6068     if (MI.isCall())
6069       return true;
6070 
6071     if (!MI.modifiesRegister(AArch64::SP, &TRI) &&
6072         !MI.readsRegister(AArch64::SP, &TRI))
6073       return true;
6074 
6075     // Any modification of SP will break our code to save/restore LR.
6076     // FIXME: We could handle some instructions which add a constant
6077     // offset to SP, with a bit more work.
6078     if (MI.modifiesRegister(AArch64::SP, &TRI))
6079       return false;
6080 
6081     // At this point, we have a stack instruction that we might need to
6082     // fix up. We'll handle it if it's a load or store.
6083     if (MI.mayLoadOrStore()) {
6084       const MachineOperand *Base; // Filled with the base operand of MI.
6085       int64_t Offset;             // Filled with the offset of MI.
6086       bool OffsetIsScalable;
6087 
6088       // Does it allow us to offset the base operand and is the base the
6089       // register SP?
6090       if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable, &TRI) ||
6091           !Base->isReg() || Base->getReg() != AArch64::SP)
6092         return false;
6093 
6094       // Fixe-up code below assumes bytes.
6095       if (OffsetIsScalable)
6096         return false;
6097 
6098       // Find the minimum/maximum offset for this instruction and check
6099       // if fixing it up would be in range.
6100       int64_t MinOffset,
6101           MaxOffset;  // Unscaled offsets for the instruction.
6102       TypeSize Scale(0U, false); // The scale to multiply the offsets by.
6103       unsigned DummyWidth;
6104       getMemOpInfo(MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
6105 
6106       Offset += 16; // Update the offset to what it would be if we outlined.
6107       if (Offset < MinOffset * (int64_t)Scale.getFixedSize() ||
6108           Offset > MaxOffset * (int64_t)Scale.getFixedSize())
6109         return false;
6110 
6111       // It's in range, so we can outline it.
6112       return true;
6113     }
6114 
6115     // FIXME: Add handling for instructions like "add x0, sp, #8".
6116 
6117     // We can't fix it up, so don't outline it.
6118     return false;
6119   };
6120 
6121   // True if it's possible to fix up each stack instruction in this sequence.
6122   // Important for frames/call variants that modify the stack.
6123   bool AllStackInstrsSafe = std::all_of(
6124       FirstCand.front(), std::next(FirstCand.back()), IsSafeToFixup);
6125 
6126   // If the last instruction in any candidate is a terminator, then we should
6127   // tail call all of the candidates.
6128   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
6129     FrameID = MachineOutlinerTailCall;
6130     NumBytesToCreateFrame = 0;
6131     SetCandidateCallInfo(MachineOutlinerTailCall, 4);
6132   }
6133 
6134   else if (LastInstrOpcode == AArch64::BL ||
6135            ((LastInstrOpcode == AArch64::BLR ||
6136              LastInstrOpcode == AArch64::BLRNoIP) &&
6137             !HasBTI)) {
6138     // FIXME: Do we need to check if the code after this uses the value of LR?
6139     FrameID = MachineOutlinerThunk;
6140     NumBytesToCreateFrame = 0;
6141     SetCandidateCallInfo(MachineOutlinerThunk, 4);
6142   }
6143 
6144   else {
6145     // We need to decide how to emit calls + frames. We can always emit the same
6146     // frame if we don't need to save to the stack. If we have to save to the
6147     // stack, then we need a different frame.
6148     unsigned NumBytesNoStackCalls = 0;
6149     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
6150 
6151     // Check if we have to save LR.
6152     for (outliner::Candidate &C : RepeatedSequenceLocs) {
6153       C.initLRU(TRI);
6154 
6155       // If we have a noreturn caller, then we're going to be conservative and
6156       // say that we have to save LR. If we don't have a ret at the end of the
6157       // block, then we can't reason about liveness accurately.
6158       //
6159       // FIXME: We can probably do better than always disabling this in
6160       // noreturn functions by fixing up the liveness info.
6161       bool IsNoReturn =
6162           C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
6163 
6164       // Is LR available? If so, we don't need a save.
6165       if (C.LRU.available(AArch64::LR) && !IsNoReturn) {
6166         NumBytesNoStackCalls += 4;
6167         C.setCallInfo(MachineOutlinerNoLRSave, 4);
6168         CandidatesWithoutStackFixups.push_back(C);
6169       }
6170 
6171       // Is an unused register available? If so, we won't modify the stack, so
6172       // we can outline with the same frame type as those that don't save LR.
6173       else if (findRegisterToSaveLRTo(C)) {
6174         NumBytesNoStackCalls += 12;
6175         C.setCallInfo(MachineOutlinerRegSave, 12);
6176         CandidatesWithoutStackFixups.push_back(C);
6177       }
6178 
6179       // Is SP used in the sequence at all? If not, we don't have to modify
6180       // the stack, so we are guaranteed to get the same frame.
6181       else if (C.UsedInSequence.available(AArch64::SP)) {
6182         NumBytesNoStackCalls += 12;
6183         C.setCallInfo(MachineOutlinerDefault, 12);
6184         CandidatesWithoutStackFixups.push_back(C);
6185       }
6186 
6187       // If we outline this, we need to modify the stack. Pretend we don't
6188       // outline this by saving all of its bytes.
6189       else {
6190         NumBytesNoStackCalls += SequenceSize;
6191       }
6192     }
6193 
6194     // If there are no places where we have to save LR, then note that we
6195     // don't have to update the stack. Otherwise, give every candidate the
6196     // default call type, as long as it's safe to do so.
6197     if (!AllStackInstrsSafe ||
6198         NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
6199       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
6200       FrameID = MachineOutlinerNoLRSave;
6201     } else {
6202       SetCandidateCallInfo(MachineOutlinerDefault, 12);
6203 
6204       // Bugzilla ID: 46767
6205       // TODO: Check if fixing up the stack more than once is safe so we can
6206       // outline these.
6207       //
6208       // An outline resulting in a caller that requires stack fixups at the
6209       // callsite to a callee that also requires stack fixups can happen when
6210       // there are no available registers at the candidate callsite for a
6211       // candidate that itself also has calls.
6212       //
6213       // In other words if function_containing_sequence in the following pseudo
6214       // assembly requires that we save LR at the point of the call, but there
6215       // are no available registers: in this case we save using SP and as a
6216       // result the SP offsets requires stack fixups by multiples of 16.
6217       //
6218       // function_containing_sequence:
6219       //   ...
6220       //   save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
6221       //   call OUTLINED_FUNCTION_N
6222       //   restore LR from SP
6223       //   ...
6224       //
6225       // OUTLINED_FUNCTION_N:
6226       //   save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
6227       //   ...
6228       //   bl foo
6229       //   restore LR from SP
6230       //   ret
6231       //
6232       // Because the code to handle more than one stack fixup does not
6233       // currently have the proper checks for legality, these cases will assert
6234       // in the AArch64 MachineOutliner. This is because the code to do this
6235       // needs more hardening, testing, better checks that generated code is
6236       // legal, etc and because it is only verified to handle a single pass of
6237       // stack fixup.
6238       //
6239       // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
6240       // these cases until they are known to be handled. Bugzilla 46767 is
6241       // referenced in comments at the assert site.
6242       //
6243       // To avoid asserting (or generating non-legal code on noassert builds)
6244       // we remove all candidates which would need more than one stack fixup by
6245       // pruning the cases where the candidate has calls while also having no
6246       // available LR and having no available general purpose registers to copy
6247       // LR to (ie one extra stack save/restore).
6248       //
6249       if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6250         erase_if(RepeatedSequenceLocs, [this](outliner::Candidate &C) {
6251           return (std::any_of(
6252                      C.front(), std::next(C.back()),
6253                      [](const MachineInstr &MI) { return MI.isCall(); })) &&
6254                  (!C.LRU.available(AArch64::LR) || !findRegisterToSaveLRTo(C));
6255         });
6256       }
6257     }
6258 
6259     // If we dropped all of the candidates, bail out here.
6260     if (RepeatedSequenceLocs.size() < 2) {
6261       RepeatedSequenceLocs.clear();
6262       return outliner::OutlinedFunction();
6263     }
6264   }
6265 
6266   // Does every candidate's MBB contain a call? If so, then we might have a call
6267   // in the range.
6268   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6269     // Check if the range contains a call. These require a save + restore of the
6270     // link register.
6271     bool ModStackToSaveLR = false;
6272     if (std::any_of(FirstCand.front(), FirstCand.back(),
6273                     [](const MachineInstr &MI) { return MI.isCall(); }))
6274       ModStackToSaveLR = true;
6275 
6276     // Handle the last instruction separately. If this is a tail call, then the
6277     // last instruction is a call. We don't want to save + restore in this case.
6278     // However, it could be possible that the last instruction is a call without
6279     // it being valid to tail call this sequence. We should consider this as
6280     // well.
6281     else if (FrameID != MachineOutlinerThunk &&
6282              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6283       ModStackToSaveLR = true;
6284 
6285     if (ModStackToSaveLR) {
6286       // We can't fix up the stack. Bail out.
6287       if (!AllStackInstrsSafe) {
6288         RepeatedSequenceLocs.clear();
6289         return outliner::OutlinedFunction();
6290       }
6291 
6292       // Save + restore LR.
6293       NumBytesToCreateFrame += 8;
6294     }
6295   }
6296 
6297   // If we have CFI instructions, we can only outline if the outlined section
6298   // can be a tail call
6299   if (FrameID != MachineOutlinerTailCall && CFICount > 0)
6300     return outliner::OutlinedFunction();
6301 
6302   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6303                                     NumBytesToCreateFrame, FrameID);
6304 }
6305 
6306 bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
6307     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6308   const Function &F = MF.getFunction();
6309 
6310   // Can F be deduplicated by the linker? If it can, don't outline from it.
6311   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6312     return false;
6313 
6314   // Don't outline from functions with section markings; the program could
6315   // expect that all the code is in the named section.
6316   // FIXME: Allow outlining from multiple functions with the same section
6317   // marking.
6318   if (F.hasSection())
6319     return false;
6320 
6321   // Outlining from functions with redzones is unsafe since the outliner may
6322   // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
6323   // outline from it.
6324   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
6325   if (!AFI || AFI->hasRedZone().getValueOr(true))
6326     return false;
6327 
6328   // FIXME: Teach the outliner to generate/handle Windows unwind info.
6329   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
6330     return false;
6331 
6332   // It's safe to outline from MF.
6333   return true;
6334 }
6335 
6336 bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6337                                               unsigned &Flags) const {
6338   // Check if LR is available through all of the MBB. If it's not, then set
6339   // a flag.
6340   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6341          "Suitable Machine Function for outlining must track liveness");
6342   LiveRegUnits LRU(getRegisterInfo());
6343 
6344   std::for_each(MBB.rbegin(), MBB.rend(),
6345                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6346 
6347   // Check if each of the unsafe registers are available...
6348   bool W16AvailableInBlock = LRU.available(AArch64::W16);
6349   bool W17AvailableInBlock = LRU.available(AArch64::W17);
6350   bool NZCVAvailableInBlock = LRU.available(AArch64::NZCV);
6351 
6352   // If all of these are dead (and not live out), we know we don't have to check
6353   // them later.
6354   if (W16AvailableInBlock && W17AvailableInBlock && NZCVAvailableInBlock)
6355     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6356 
6357   // Now, add the live outs to the set.
6358   LRU.addLiveOuts(MBB);
6359 
6360   // If any of these registers is available in the MBB, but also a live out of
6361   // the block, then we know outlining is unsafe.
6362   if (W16AvailableInBlock && !LRU.available(AArch64::W16))
6363     return false;
6364   if (W17AvailableInBlock && !LRU.available(AArch64::W17))
6365     return false;
6366   if (NZCVAvailableInBlock && !LRU.available(AArch64::NZCV))
6367     return false;
6368 
6369   // Check if there's a call inside this MachineBasicBlock. If there is, then
6370   // set a flag.
6371   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6372     Flags |= MachineOutlinerMBBFlags::HasCalls;
6373 
6374   MachineFunction *MF = MBB.getParent();
6375 
6376   // In the event that we outline, we may have to save LR. If there is an
6377   // available register in the MBB, then we'll always save LR there. Check if
6378   // this is true.
6379   bool CanSaveLR = false;
6380   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
6381       MF->getSubtarget().getRegisterInfo());
6382 
6383   // Check if there is an available register across the sequence that we can
6384   // use.
6385   for (unsigned Reg : AArch64::GPR64RegClass) {
6386     if (!ARI->isReservedReg(*MF, Reg) && Reg != AArch64::LR &&
6387         Reg != AArch64::X16 && Reg != AArch64::X17 && LRU.available(Reg)) {
6388       CanSaveLR = true;
6389       break;
6390     }
6391   }
6392 
6393   // Check if we have a register we can save LR to, and if LR was used
6394   // somewhere. If both of those things are true, then we need to evaluate the
6395   // safety of outlining stack instructions later.
6396   if (!CanSaveLR && !LRU.available(AArch64::LR))
6397     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6398 
6399   return true;
6400 }
6401 
6402 outliner::InstrType
6403 AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6404                                    unsigned Flags) const {
6405   MachineInstr &MI = *MIT;
6406   MachineBasicBlock *MBB = MI.getParent();
6407   MachineFunction *MF = MBB->getParent();
6408   AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
6409 
6410   // Don't outline anything used for return address signing. The outlined
6411   // function will get signed later if needed
6412   switch (MI.getOpcode()) {
6413   case AArch64::PACIASP:
6414   case AArch64::PACIBSP:
6415   case AArch64::AUTIASP:
6416   case AArch64::AUTIBSP:
6417   case AArch64::RETAA:
6418   case AArch64::RETAB:
6419   case AArch64::EMITBKEY:
6420     return outliner::InstrType::Illegal;
6421   }
6422 
6423   // Don't outline LOHs.
6424   if (FuncInfo->getLOHRelated().count(&MI))
6425     return outliner::InstrType::Illegal;
6426 
6427   // We can only outline these if we will tail call the outlined function, or
6428   // fix up the CFI offsets. Currently, CFI instructions are outlined only if
6429   // in a tail call.
6430   //
6431   // FIXME: If the proper fixups for the offset are implemented, this should be
6432   // possible.
6433   if (MI.isCFIInstruction())
6434     return outliner::InstrType::Legal;
6435 
6436   // Don't allow debug values to impact outlining type.
6437   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6438     return outliner::InstrType::Invisible;
6439 
6440   // At this point, KILL instructions don't really tell us much so we can go
6441   // ahead and skip over them.
6442   if (MI.isKill())
6443     return outliner::InstrType::Invisible;
6444 
6445   // Is this a terminator for a basic block?
6446   if (MI.isTerminator()) {
6447 
6448     // Is this the end of a function?
6449     if (MI.getParent()->succ_empty())
6450       return outliner::InstrType::Legal;
6451 
6452     // It's not, so don't outline it.
6453     return outliner::InstrType::Illegal;
6454   }
6455 
6456   // Make sure none of the operands are un-outlinable.
6457   for (const MachineOperand &MOP : MI.operands()) {
6458     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6459         MOP.isTargetIndex())
6460       return outliner::InstrType::Illegal;
6461 
6462     // If it uses LR or W30 explicitly, then don't touch it.
6463     if (MOP.isReg() && !MOP.isImplicit() &&
6464         (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
6465       return outliner::InstrType::Illegal;
6466   }
6467 
6468   // Special cases for instructions that can always be outlined, but will fail
6469   // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
6470   // be outlined because they don't require a *specific* value to be in LR.
6471   if (MI.getOpcode() == AArch64::ADRP)
6472     return outliner::InstrType::Legal;
6473 
6474   // If MI is a call we might be able to outline it. We don't want to outline
6475   // any calls that rely on the position of items on the stack. When we outline
6476   // something containing a call, we have to emit a save and restore of LR in
6477   // the outlined function. Currently, this always happens by saving LR to the
6478   // stack. Thus, if we outline, say, half the parameters for a function call
6479   // plus the call, then we'll break the callee's expectations for the layout
6480   // of the stack.
6481   //
6482   // FIXME: Allow calls to functions which construct a stack frame, as long
6483   // as they don't access arguments on the stack.
6484   // FIXME: Figure out some way to analyze functions defined in other modules.
6485   // We should be able to compute the memory usage based on the IR calling
6486   // convention, even if we can't see the definition.
6487   if (MI.isCall()) {
6488     // Get the function associated with the call. Look at each operand and find
6489     // the one that represents the callee and get its name.
6490     const Function *Callee = nullptr;
6491     for (const MachineOperand &MOP : MI.operands()) {
6492       if (MOP.isGlobal()) {
6493         Callee = dyn_cast<Function>(MOP.getGlobal());
6494         break;
6495       }
6496     }
6497 
6498     // Never outline calls to mcount.  There isn't any rule that would require
6499     // this, but the Linux kernel's "ftrace" feature depends on it.
6500     if (Callee && Callee->getName() == "\01_mcount")
6501       return outliner::InstrType::Illegal;
6502 
6503     // If we don't know anything about the callee, assume it depends on the
6504     // stack layout of the caller. In that case, it's only legal to outline
6505     // as a tail-call. Explicitly list the call instructions we know about so we
6506     // don't get unexpected results with call pseudo-instructions.
6507     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6508     if (MI.getOpcode() == AArch64::BLR ||
6509         MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
6510       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6511 
6512     if (!Callee)
6513       return UnknownCallOutlineType;
6514 
6515     // We have a function we have information about. Check it if it's something
6516     // can safely outline.
6517     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6518 
6519     // We don't know what's going on with the callee at all. Don't touch it.
6520     if (!CalleeMF)
6521       return UnknownCallOutlineType;
6522 
6523     // Check if we know anything about the callee saves on the function. If we
6524     // don't, then don't touch it, since that implies that we haven't
6525     // computed anything about its stack frame yet.
6526     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6527     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6528         MFI.getNumObjects() > 0)
6529       return UnknownCallOutlineType;
6530 
6531     // At this point, we can say that CalleeMF ought to not pass anything on the
6532     // stack. Therefore, we can outline it.
6533     return outliner::InstrType::Legal;
6534   }
6535 
6536   // Don't outline positions.
6537   if (MI.isPosition())
6538     return outliner::InstrType::Illegal;
6539 
6540   // Don't touch the link register or W30.
6541   if (MI.readsRegister(AArch64::W30, &getRegisterInfo()) ||
6542       MI.modifiesRegister(AArch64::W30, &getRegisterInfo()))
6543     return outliner::InstrType::Illegal;
6544 
6545   // Don't outline BTI instructions, because that will prevent the outlining
6546   // site from being indirectly callable.
6547   if (MI.getOpcode() == AArch64::HINT) {
6548     int64_t Imm = MI.getOperand(0).getImm();
6549     if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
6550       return outliner::InstrType::Illegal;
6551   }
6552 
6553   return outliner::InstrType::Legal;
6554 }
6555 
6556 void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6557   for (MachineInstr &MI : MBB) {
6558     const MachineOperand *Base;
6559     unsigned Width;
6560     int64_t Offset;
6561     bool OffsetIsScalable;
6562 
6563     // Is this a load or store with an immediate offset with SP as the base?
6564     if (!MI.mayLoadOrStore() ||
6565         !getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable, Width,
6566                                       &RI) ||
6567         (Base->isReg() && Base->getReg() != AArch64::SP))
6568       continue;
6569 
6570     // It is, so we have to fix it up.
6571     TypeSize Scale(0U, false);
6572     int64_t Dummy1, Dummy2;
6573 
6574     MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(MI);
6575     assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
6576     getMemOpInfo(MI.getOpcode(), Scale, Width, Dummy1, Dummy2);
6577     assert(Scale != 0 && "Unexpected opcode!");
6578     assert(!OffsetIsScalable && "Expected offset to be a byte offset");
6579 
6580     // We've pushed the return address to the stack, so add 16 to the offset.
6581     // This is safe, since we already checked if it would overflow when we
6582     // checked if this instruction was legal to outline.
6583     int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedSize();
6584     StackOffsetOperand.setImm(NewImm);
6585   }
6586 }
6587 
6588 static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
6589                                  bool ShouldSignReturnAddr,
6590                                  bool ShouldSignReturnAddrWithAKey) {
6591   if (ShouldSignReturnAddr) {
6592     MachineBasicBlock::iterator MBBPAC = MBB.begin();
6593     MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator();
6594     const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
6595     const TargetInstrInfo *TII = Subtarget.getInstrInfo();
6596     DebugLoc DL;
6597 
6598     if (MBBAUT != MBB.end())
6599       DL = MBBAUT->getDebugLoc();
6600 
6601     // At the very beginning of the basic block we insert the following
6602     // depending on the key type
6603     //
6604     // a_key:                   b_key:
6605     //    PACIASP                   EMITBKEY
6606     //    CFI_INSTRUCTION           PACIBSP
6607     //                              CFI_INSTRUCTION
6608     if (ShouldSignReturnAddrWithAKey) {
6609       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIASP))
6610           .setMIFlag(MachineInstr::FrameSetup);
6611     } else {
6612       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::EMITBKEY))
6613           .setMIFlag(MachineInstr::FrameSetup);
6614       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIBSP))
6615           .setMIFlag(MachineInstr::FrameSetup);
6616     }
6617     unsigned CFIIndex =
6618         MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
6619     BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::CFI_INSTRUCTION))
6620         .addCFIIndex(CFIIndex)
6621         .setMIFlags(MachineInstr::FrameSetup);
6622 
6623     // If v8.3a features are available we can replace a RET instruction by
6624     // RETAA or RETAB and omit the AUT instructions
6625     if (Subtarget.hasV8_3aOps() && MBBAUT != MBB.end() &&
6626         MBBAUT->getOpcode() == AArch64::RET) {
6627       BuildMI(MBB, MBBAUT, DL,
6628               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6629                                                     : AArch64::RETAB))
6630           .copyImplicitOps(*MBBAUT);
6631       MBB.erase(MBBAUT);
6632     } else {
6633       BuildMI(MBB, MBBAUT, DL,
6634               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6635                                                     : AArch64::AUTIBSP))
6636           .setMIFlag(MachineInstr::FrameDestroy);
6637     }
6638   }
6639 }
6640 
6641 void AArch64InstrInfo::buildOutlinedFrame(
6642     MachineBasicBlock &MBB, MachineFunction &MF,
6643     const outliner::OutlinedFunction &OF) const {
6644 
6645   AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
6646 
6647   if (OF.FrameConstructionID == MachineOutlinerTailCall)
6648     FI->setOutliningStyle("Tail Call");
6649   else if (OF.FrameConstructionID == MachineOutlinerThunk) {
6650     // For thunk outlining, rewrite the last instruction from a call to a
6651     // tail-call.
6652     MachineInstr *Call = &*--MBB.instr_end();
6653     unsigned TailOpcode;
6654     if (Call->getOpcode() == AArch64::BL) {
6655       TailOpcode = AArch64::TCRETURNdi;
6656     } else {
6657       assert(Call->getOpcode() == AArch64::BLR ||
6658              Call->getOpcode() == AArch64::BLRNoIP);
6659       TailOpcode = AArch64::TCRETURNriALL;
6660     }
6661     MachineInstr *TC = BuildMI(MF, DebugLoc(), get(TailOpcode))
6662                            .add(Call->getOperand(0))
6663                            .addImm(0);
6664     MBB.insert(MBB.end(), TC);
6665     Call->eraseFromParent();
6666 
6667     FI->setOutliningStyle("Thunk");
6668   }
6669 
6670   bool IsLeafFunction = true;
6671 
6672   // Is there a call in the outlined range?
6673   auto IsNonTailCall = [](const MachineInstr &MI) {
6674     return MI.isCall() && !MI.isReturn();
6675   };
6676 
6677   if (std::any_of(MBB.instr_begin(), MBB.instr_end(), IsNonTailCall)) {
6678     // Fix up the instructions in the range, since we're going to modify the
6679     // stack.
6680 
6681     // Bugzilla ID: 46767
6682     // TODO: Check if fixing up twice is safe so we can outline these.
6683     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6684            "Can only fix up stack references once");
6685     fixupPostOutline(MBB);
6686 
6687     IsLeafFunction = false;
6688 
6689     // LR has to be a live in so that we can save it.
6690     if (!MBB.isLiveIn(AArch64::LR))
6691       MBB.addLiveIn(AArch64::LR);
6692 
6693     MachineBasicBlock::iterator It = MBB.begin();
6694     MachineBasicBlock::iterator Et = MBB.end();
6695 
6696     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6697         OF.FrameConstructionID == MachineOutlinerThunk)
6698       Et = std::prev(MBB.end());
6699 
6700     // Insert a save before the outlined region
6701     MachineInstr *STRXpre = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6702                                 .addReg(AArch64::SP, RegState::Define)
6703                                 .addReg(AArch64::LR)
6704                                 .addReg(AArch64::SP)
6705                                 .addImm(-16);
6706     It = MBB.insert(It, STRXpre);
6707 
6708     const TargetSubtargetInfo &STI = MF.getSubtarget();
6709     const MCRegisterInfo *MRI = STI.getRegisterInfo();
6710     unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true);
6711 
6712     // Add a CFI saying the stack was moved 16 B down.
6713     int64_t StackPosEntry =
6714         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 16));
6715     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6716         .addCFIIndex(StackPosEntry)
6717         .setMIFlags(MachineInstr::FrameSetup);
6718 
6719     // Add a CFI saying that the LR that we want to find is now 16 B higher than
6720     // before.
6721     int64_t LRPosEntry =
6722         MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfReg, -16));
6723     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6724         .addCFIIndex(LRPosEntry)
6725         .setMIFlags(MachineInstr::FrameSetup);
6726 
6727     // Insert a restore before the terminator for the function.
6728     MachineInstr *LDRXpost = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6729                                  .addReg(AArch64::SP, RegState::Define)
6730                                  .addReg(AArch64::LR, RegState::Define)
6731                                  .addReg(AArch64::SP)
6732                                  .addImm(16);
6733     Et = MBB.insert(Et, LDRXpost);
6734   }
6735 
6736   // If a bunch of candidates reach this point they must agree on their return
6737   // address signing. It is therefore enough to just consider the signing
6738   // behaviour of one of them
6739   const Function &CF = OF.Candidates.front().getMF()->getFunction();
6740   bool ShouldSignReturnAddr = false;
6741   if (CF.hasFnAttribute("sign-return-address")) {
6742     StringRef Scope =
6743         CF.getFnAttribute("sign-return-address").getValueAsString();
6744     if (Scope.equals("all"))
6745       ShouldSignReturnAddr = true;
6746     else if (Scope.equals("non-leaf") && !IsLeafFunction)
6747       ShouldSignReturnAddr = true;
6748   }
6749 
6750   // a_key is the default
6751   bool ShouldSignReturnAddrWithAKey = true;
6752   if (CF.hasFnAttribute("sign-return-address-key")) {
6753     const StringRef Key =
6754         CF.getFnAttribute("sign-return-address-key").getValueAsString();
6755     // Key can either be a_key or b_key
6756     assert((Key.equals_lower("a_key") || Key.equals_lower("b_key")) &&
6757            "Return address signing key must be either a_key or b_key");
6758     ShouldSignReturnAddrWithAKey = Key.equals_lower("a_key");
6759   }
6760 
6761   // If this is a tail call outlined function, then there's already a return.
6762   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6763       OF.FrameConstructionID == MachineOutlinerThunk) {
6764     signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6765                          ShouldSignReturnAddrWithAKey);
6766     return;
6767   }
6768 
6769   // It's not a tail call, so we have to insert the return ourselves.
6770 
6771   // LR has to be a live in so that we can return to it.
6772   if (!MBB.isLiveIn(AArch64::LR))
6773     MBB.addLiveIn(AArch64::LR);
6774 
6775   MachineInstr *ret = BuildMI(MF, DebugLoc(), get(AArch64::RET))
6776                           .addReg(AArch64::LR);
6777   MBB.insert(MBB.end(), ret);
6778 
6779   signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6780                        ShouldSignReturnAddrWithAKey);
6781 
6782   FI->setOutliningStyle("Function");
6783 
6784   // Did we have to modify the stack by saving the link register?
6785   if (OF.FrameConstructionID != MachineOutlinerDefault)
6786     return;
6787 
6788   // We modified the stack.
6789   // Walk over the basic block and fix up all the stack accesses.
6790   fixupPostOutline(MBB);
6791 }
6792 
6793 MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
6794     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6795     MachineFunction &MF, const outliner::Candidate &C) const {
6796 
6797   // Are we tail calling?
6798   if (C.CallConstructionID == MachineOutlinerTailCall) {
6799     // If yes, then we can just branch to the label.
6800     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::TCRETURNdi))
6801                             .addGlobalAddress(M.getNamedValue(MF.getName()))
6802                             .addImm(0));
6803     return It;
6804   }
6805 
6806   // Are we saving the link register?
6807   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6808       C.CallConstructionID == MachineOutlinerThunk) {
6809     // No, so just insert the call.
6810     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6811                             .addGlobalAddress(M.getNamedValue(MF.getName())));
6812     return It;
6813   }
6814 
6815   // We want to return the spot where we inserted the call.
6816   MachineBasicBlock::iterator CallPt;
6817 
6818   // Instructions for saving and restoring LR around the call instruction we're
6819   // going to insert.
6820   MachineInstr *Save;
6821   MachineInstr *Restore;
6822   // Can we save to a register?
6823   if (C.CallConstructionID == MachineOutlinerRegSave) {
6824     // FIXME: This logic should be sunk into a target-specific interface so that
6825     // we don't have to recompute the register.
6826     unsigned Reg = findRegisterToSaveLRTo(C);
6827     assert(Reg != 0 && "No callee-saved register available?");
6828 
6829     // Save and restore LR from that register.
6830     Save = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), Reg)
6831                .addReg(AArch64::XZR)
6832                .addReg(AArch64::LR)
6833                .addImm(0);
6834     Restore = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
6835                 .addReg(AArch64::XZR)
6836                 .addReg(Reg)
6837                 .addImm(0);
6838   } else {
6839     // We have the default case. Save and restore from SP.
6840     Save = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6841                .addReg(AArch64::SP, RegState::Define)
6842                .addReg(AArch64::LR)
6843                .addReg(AArch64::SP)
6844                .addImm(-16);
6845     Restore = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6846                   .addReg(AArch64::SP, RegState::Define)
6847                   .addReg(AArch64::LR, RegState::Define)
6848                   .addReg(AArch64::SP)
6849                   .addImm(16);
6850   }
6851 
6852   It = MBB.insert(It, Save);
6853   It++;
6854 
6855   // Insert the call.
6856   It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6857                           .addGlobalAddress(M.getNamedValue(MF.getName())));
6858   CallPt = It;
6859   It++;
6860 
6861   It = MBB.insert(It, Restore);
6862   return CallPt;
6863 }
6864 
6865 bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
6866   MachineFunction &MF) const {
6867   return MF.getFunction().hasMinSize();
6868 }
6869 
6870 Optional<DestSourcePair>
6871 AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
6872 
6873   // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
6874   // and zero immediate operands used as an alias for mov instruction.
6875   if (MI.getOpcode() == AArch64::ORRWrs &&
6876       MI.getOperand(1).getReg() == AArch64::WZR &&
6877       MI.getOperand(3).getImm() == 0x0) {
6878     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6879   }
6880 
6881   if (MI.getOpcode() == AArch64::ORRXrs &&
6882       MI.getOperand(1).getReg() == AArch64::XZR &&
6883       MI.getOperand(3).getImm() == 0x0) {
6884     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6885   }
6886 
6887   return None;
6888 }
6889 
6890 Optional<RegImmPair> AArch64InstrInfo::isAddImmediate(const MachineInstr &MI,
6891                                                       Register Reg) const {
6892   int Sign = 1;
6893   int64_t Offset = 0;
6894 
6895   // TODO: Handle cases where Reg is a super- or sub-register of the
6896   // destination register.
6897   const MachineOperand &Op0 = MI.getOperand(0);
6898   if (!Op0.isReg() || Reg != Op0.getReg())
6899     return None;
6900 
6901   switch (MI.getOpcode()) {
6902   default:
6903     return None;
6904   case AArch64::SUBWri:
6905   case AArch64::SUBXri:
6906   case AArch64::SUBSWri:
6907   case AArch64::SUBSXri:
6908     Sign *= -1;
6909     LLVM_FALLTHROUGH;
6910   case AArch64::ADDSWri:
6911   case AArch64::ADDSXri:
6912   case AArch64::ADDWri:
6913   case AArch64::ADDXri: {
6914     // TODO: Third operand can be global address (usually some string).
6915     if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg() ||
6916         !MI.getOperand(2).isImm())
6917       return None;
6918     int Shift = MI.getOperand(3).getImm();
6919     assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
6920     Offset = Sign * (MI.getOperand(2).getImm() << Shift);
6921   }
6922   }
6923   return RegImmPair{MI.getOperand(1).getReg(), Offset};
6924 }
6925 
6926 /// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
6927 /// the destination register then, if possible, describe the value in terms of
6928 /// the source register.
6929 static Optional<ParamLoadedValue>
6930 describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
6931                        const TargetInstrInfo *TII,
6932                        const TargetRegisterInfo *TRI) {
6933   auto DestSrc = TII->isCopyInstr(MI);
6934   if (!DestSrc)
6935     return None;
6936 
6937   Register DestReg = DestSrc->Destination->getReg();
6938   Register SrcReg = DestSrc->Source->getReg();
6939 
6940   auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
6941 
6942   // If the described register is the destination, just return the source.
6943   if (DestReg == DescribedReg)
6944     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6945 
6946   // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
6947   if (MI.getOpcode() == AArch64::ORRWrs &&
6948       TRI->isSuperRegister(DestReg, DescribedReg))
6949     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6950 
6951   // We may need to describe the lower part of a ORRXrs move.
6952   if (MI.getOpcode() == AArch64::ORRXrs &&
6953       TRI->isSubRegister(DestReg, DescribedReg)) {
6954     Register SrcSubReg = TRI->getSubReg(SrcReg, AArch64::sub_32);
6955     return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr);
6956   }
6957 
6958   assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
6959          "Unhandled ORR[XW]rs copy case");
6960 
6961   return None;
6962 }
6963 
6964 Optional<ParamLoadedValue>
6965 AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
6966                                       Register Reg) const {
6967   const MachineFunction *MF = MI.getMF();
6968   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6969   switch (MI.getOpcode()) {
6970   case AArch64::MOVZWi:
6971   case AArch64::MOVZXi: {
6972     // MOVZWi may be used for producing zero-extended 32-bit immediates in
6973     // 64-bit parameters, so we need to consider super-registers.
6974     if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
6975       return None;
6976 
6977     if (!MI.getOperand(1).isImm())
6978       return None;
6979     int64_t Immediate = MI.getOperand(1).getImm();
6980     int Shift = MI.getOperand(2).getImm();
6981     return ParamLoadedValue(MachineOperand::CreateImm(Immediate << Shift),
6982                             nullptr);
6983   }
6984   case AArch64::ORRWrs:
6985   case AArch64::ORRXrs:
6986     return describeORRLoadedValue(MI, Reg, this, TRI);
6987   }
6988 
6989   return TargetInstrInfo::describeLoadedValue(MI, Reg);
6990 }
6991 
6992 uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
6993   return get(Opc).TSFlags & AArch64::ElementSizeMask;
6994 }
6995 
6996 unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
6997   if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
6998     return AArch64::BLRNoIP;
6999   else
7000     return AArch64::BLR;
7001 }
7002 
7003 #define GET_INSTRINFO_HELPERS
7004 #define GET_INSTRMAP_INFO
7005 #include "AArch64GenInstrInfo.inc"
7006