1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64InstrInfo.h"
14 #include "AArch64MachineFunctionInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "Utils/AArch64BaseInfo.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineModuleInfo.h"
28 #include "llvm/CodeGen/MachineOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetRegisterInfo.h"
32 #include "llvm/CodeGen/TargetSubtargetInfo.h"
33 #include "llvm/IR/DebugInfoMetadata.h"
34 #include "llvm/IR/DebugLoc.h"
35 #include "llvm/IR/GlobalValue.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/MCInstrDesc.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Compiler.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <cassert>
48 #include <cstdint>
49 #include <iterator>
50 #include <utility>
51 
52 using namespace llvm;
53 
54 #define GET_INSTRINFO_CTOR_DTOR
55 #include "AArch64GenInstrInfo.inc"
56 
57 static cl::opt<unsigned> TBZDisplacementBits(
58     "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
59     cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
60 
61 static cl::opt<unsigned> CBZDisplacementBits(
62     "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
63     cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
64 
65 static cl::opt<unsigned>
66     BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
67                         cl::desc("Restrict range of Bcc instructions (DEBUG)"));
68 
69 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
70     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
71                           AArch64::CATCHRET),
72       RI(STI.getTargetTriple()), Subtarget(STI) {}
73 
74 /// GetInstSize - Return the number of bytes of code the specified
75 /// instruction may be.  This returns the maximum number of bytes.
76 unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
77   const MachineBasicBlock &MBB = *MI.getParent();
78   const MachineFunction *MF = MBB.getParent();
79   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
80 
81   {
82     auto Op = MI.getOpcode();
83     if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
84       return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
85   }
86 
87   // Meta-instructions emit no code.
88   if (MI.isMetaInstruction())
89     return 0;
90 
91   // FIXME: We currently only handle pseudoinstructions that don't get expanded
92   //        before the assembly printer.
93   unsigned NumBytes = 0;
94   const MCInstrDesc &Desc = MI.getDesc();
95   switch (Desc.getOpcode()) {
96   default:
97     // Anything not explicitly designated otherwise is a normal 4-byte insn.
98     NumBytes = 4;
99     break;
100   case TargetOpcode::STACKMAP:
101     // The upper bound for a stackmap intrinsic is the full length of its shadow
102     NumBytes = StackMapOpers(&MI).getNumPatchBytes();
103     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
104     break;
105   case TargetOpcode::PATCHPOINT:
106     // The size of the patchpoint intrinsic is the number of bytes requested
107     NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
108     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
109     break;
110   case AArch64::TLSDESC_CALLSEQ:
111     // This gets lowered to an instruction sequence which takes 16 bytes
112     NumBytes = 16;
113     break;
114   case AArch64::JumpTableDest32:
115   case AArch64::JumpTableDest16:
116   case AArch64::JumpTableDest8:
117     NumBytes = 12;
118     break;
119   case AArch64::SPACE:
120     NumBytes = MI.getOperand(1).getImm();
121     break;
122   case TargetOpcode::BUNDLE:
123     NumBytes = getInstBundleLength(MI);
124     break;
125   }
126 
127   return NumBytes;
128 }
129 
130 unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
131   unsigned Size = 0;
132   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
133   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
134   while (++I != E && I->isInsideBundle()) {
135     assert(!I->isBundle() && "No nested bundle!");
136     Size += getInstSizeInBytes(*I);
137   }
138   return Size;
139 }
140 
141 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
142                             SmallVectorImpl<MachineOperand> &Cond) {
143   // Block ends with fall-through condbranch.
144   switch (LastInst->getOpcode()) {
145   default:
146     llvm_unreachable("Unknown branch instruction?");
147   case AArch64::Bcc:
148     Target = LastInst->getOperand(1).getMBB();
149     Cond.push_back(LastInst->getOperand(0));
150     break;
151   case AArch64::CBZW:
152   case AArch64::CBZX:
153   case AArch64::CBNZW:
154   case AArch64::CBNZX:
155     Target = LastInst->getOperand(1).getMBB();
156     Cond.push_back(MachineOperand::CreateImm(-1));
157     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
158     Cond.push_back(LastInst->getOperand(0));
159     break;
160   case AArch64::TBZW:
161   case AArch64::TBZX:
162   case AArch64::TBNZW:
163   case AArch64::TBNZX:
164     Target = LastInst->getOperand(2).getMBB();
165     Cond.push_back(MachineOperand::CreateImm(-1));
166     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
167     Cond.push_back(LastInst->getOperand(0));
168     Cond.push_back(LastInst->getOperand(1));
169   }
170 }
171 
172 static unsigned getBranchDisplacementBits(unsigned Opc) {
173   switch (Opc) {
174   default:
175     llvm_unreachable("unexpected opcode!");
176   case AArch64::B:
177     return 64;
178   case AArch64::TBNZW:
179   case AArch64::TBZW:
180   case AArch64::TBNZX:
181   case AArch64::TBZX:
182     return TBZDisplacementBits;
183   case AArch64::CBNZW:
184   case AArch64::CBZW:
185   case AArch64::CBNZX:
186   case AArch64::CBZX:
187     return CBZDisplacementBits;
188   case AArch64::Bcc:
189     return BCCDisplacementBits;
190   }
191 }
192 
193 bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
194                                              int64_t BrOffset) const {
195   unsigned Bits = getBranchDisplacementBits(BranchOp);
196   assert(Bits >= 3 && "max branch displacement must be enough to jump"
197                       "over conditional branch expansion");
198   return isIntN(Bits, BrOffset / 4);
199 }
200 
201 MachineBasicBlock *
202 AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
203   switch (MI.getOpcode()) {
204   default:
205     llvm_unreachable("unexpected opcode!");
206   case AArch64::B:
207     return MI.getOperand(0).getMBB();
208   case AArch64::TBZW:
209   case AArch64::TBNZW:
210   case AArch64::TBZX:
211   case AArch64::TBNZX:
212     return MI.getOperand(2).getMBB();
213   case AArch64::CBZW:
214   case AArch64::CBNZW:
215   case AArch64::CBZX:
216   case AArch64::CBNZX:
217   case AArch64::Bcc:
218     return MI.getOperand(1).getMBB();
219   }
220 }
221 
222 // Branch analysis.
223 bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
224                                      MachineBasicBlock *&TBB,
225                                      MachineBasicBlock *&FBB,
226                                      SmallVectorImpl<MachineOperand> &Cond,
227                                      bool AllowModify) const {
228   // If the block has no terminators, it just falls into the block after it.
229   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
230   if (I == MBB.end())
231     return false;
232 
233   if (!isUnpredicatedTerminator(*I))
234     return false;
235 
236   // Get the last instruction in the block.
237   MachineInstr *LastInst = &*I;
238 
239   // If there is only one terminator instruction, process it.
240   unsigned LastOpc = LastInst->getOpcode();
241   if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
242     if (isUncondBranchOpcode(LastOpc)) {
243       TBB = LastInst->getOperand(0).getMBB();
244       return false;
245     }
246     if (isCondBranchOpcode(LastOpc)) {
247       // Block ends with fall-through condbranch.
248       parseCondBranch(LastInst, TBB, Cond);
249       return false;
250     }
251     return true; // Can't handle indirect branch.
252   }
253 
254   // Get the instruction before it if it is a terminator.
255   MachineInstr *SecondLastInst = &*I;
256   unsigned SecondLastOpc = SecondLastInst->getOpcode();
257 
258   // If AllowModify is true and the block ends with two or more unconditional
259   // branches, delete all but the first unconditional branch.
260   if (AllowModify && isUncondBranchOpcode(LastOpc)) {
261     while (isUncondBranchOpcode(SecondLastOpc)) {
262       LastInst->eraseFromParent();
263       LastInst = SecondLastInst;
264       LastOpc = LastInst->getOpcode();
265       if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
266         // Return now the only terminator is an unconditional branch.
267         TBB = LastInst->getOperand(0).getMBB();
268         return false;
269       } else {
270         SecondLastInst = &*I;
271         SecondLastOpc = SecondLastInst->getOpcode();
272       }
273     }
274   }
275 
276   // If there are three terminators, we don't know what sort of block this is.
277   if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
278     return true;
279 
280   // If the block ends with a B and a Bcc, handle it.
281   if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
282     parseCondBranch(SecondLastInst, TBB, Cond);
283     FBB = LastInst->getOperand(0).getMBB();
284     return false;
285   }
286 
287   // If the block ends with two unconditional branches, handle it.  The second
288   // one is not executed, so remove it.
289   if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
290     TBB = SecondLastInst->getOperand(0).getMBB();
291     I = LastInst;
292     if (AllowModify)
293       I->eraseFromParent();
294     return false;
295   }
296 
297   // ...likewise if it ends with an indirect branch followed by an unconditional
298   // branch.
299   if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
300     I = LastInst;
301     if (AllowModify)
302       I->eraseFromParent();
303     return true;
304   }
305 
306   // Otherwise, can't handle this.
307   return true;
308 }
309 
310 bool AArch64InstrInfo::reverseBranchCondition(
311     SmallVectorImpl<MachineOperand> &Cond) const {
312   if (Cond[0].getImm() != -1) {
313     // Regular Bcc
314     AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
315     Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
316   } else {
317     // Folded compare-and-branch
318     switch (Cond[1].getImm()) {
319     default:
320       llvm_unreachable("Unknown conditional branch!");
321     case AArch64::CBZW:
322       Cond[1].setImm(AArch64::CBNZW);
323       break;
324     case AArch64::CBNZW:
325       Cond[1].setImm(AArch64::CBZW);
326       break;
327     case AArch64::CBZX:
328       Cond[1].setImm(AArch64::CBNZX);
329       break;
330     case AArch64::CBNZX:
331       Cond[1].setImm(AArch64::CBZX);
332       break;
333     case AArch64::TBZW:
334       Cond[1].setImm(AArch64::TBNZW);
335       break;
336     case AArch64::TBNZW:
337       Cond[1].setImm(AArch64::TBZW);
338       break;
339     case AArch64::TBZX:
340       Cond[1].setImm(AArch64::TBNZX);
341       break;
342     case AArch64::TBNZX:
343       Cond[1].setImm(AArch64::TBZX);
344       break;
345     }
346   }
347 
348   return false;
349 }
350 
351 unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
352                                         int *BytesRemoved) const {
353   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
354   if (I == MBB.end())
355     return 0;
356 
357   if (!isUncondBranchOpcode(I->getOpcode()) &&
358       !isCondBranchOpcode(I->getOpcode()))
359     return 0;
360 
361   // Remove the branch.
362   I->eraseFromParent();
363 
364   I = MBB.end();
365 
366   if (I == MBB.begin()) {
367     if (BytesRemoved)
368       *BytesRemoved = 4;
369     return 1;
370   }
371   --I;
372   if (!isCondBranchOpcode(I->getOpcode())) {
373     if (BytesRemoved)
374       *BytesRemoved = 4;
375     return 1;
376   }
377 
378   // Remove the branch.
379   I->eraseFromParent();
380   if (BytesRemoved)
381     *BytesRemoved = 8;
382 
383   return 2;
384 }
385 
386 void AArch64InstrInfo::instantiateCondBranch(
387     MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
388     ArrayRef<MachineOperand> Cond) const {
389   if (Cond[0].getImm() != -1) {
390     // Regular Bcc
391     BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
392   } else {
393     // Folded compare-and-branch
394     // Note that we use addOperand instead of addReg to keep the flags.
395     const MachineInstrBuilder MIB =
396         BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
397     if (Cond.size() > 3)
398       MIB.addImm(Cond[3].getImm());
399     MIB.addMBB(TBB);
400   }
401 }
402 
403 unsigned AArch64InstrInfo::insertBranch(
404     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
405     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
406   // Shouldn't be a fall through.
407   assert(TBB && "insertBranch must not be told to insert a fallthrough");
408 
409   if (!FBB) {
410     if (Cond.empty()) // Unconditional branch?
411       BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
412     else
413       instantiateCondBranch(MBB, DL, TBB, Cond);
414 
415     if (BytesAdded)
416       *BytesAdded = 4;
417 
418     return 1;
419   }
420 
421   // Two-way conditional branch.
422   instantiateCondBranch(MBB, DL, TBB, Cond);
423   BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
424 
425   if (BytesAdded)
426     *BytesAdded = 8;
427 
428   return 2;
429 }
430 
431 // Find the original register that VReg is copied from.
432 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
433   while (Register::isVirtualRegister(VReg)) {
434     const MachineInstr *DefMI = MRI.getVRegDef(VReg);
435     if (!DefMI->isFullCopy())
436       return VReg;
437     VReg = DefMI->getOperand(1).getReg();
438   }
439   return VReg;
440 }
441 
442 // Determine if VReg is defined by an instruction that can be folded into a
443 // csel instruction. If so, return the folded opcode, and the replacement
444 // register.
445 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
446                                 unsigned *NewVReg = nullptr) {
447   VReg = removeCopies(MRI, VReg);
448   if (!Register::isVirtualRegister(VReg))
449     return 0;
450 
451   bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
452   const MachineInstr *DefMI = MRI.getVRegDef(VReg);
453   unsigned Opc = 0;
454   unsigned SrcOpNum = 0;
455   switch (DefMI->getOpcode()) {
456   case AArch64::ADDSXri:
457   case AArch64::ADDSWri:
458     // if NZCV is used, do not fold.
459     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
460       return 0;
461     // fall-through to ADDXri and ADDWri.
462     LLVM_FALLTHROUGH;
463   case AArch64::ADDXri:
464   case AArch64::ADDWri:
465     // add x, 1 -> csinc.
466     if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
467         DefMI->getOperand(3).getImm() != 0)
468       return 0;
469     SrcOpNum = 1;
470     Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
471     break;
472 
473   case AArch64::ORNXrr:
474   case AArch64::ORNWrr: {
475     // not x -> csinv, represented as orn dst, xzr, src.
476     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
477     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
478       return 0;
479     SrcOpNum = 2;
480     Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
481     break;
482   }
483 
484   case AArch64::SUBSXrr:
485   case AArch64::SUBSWrr:
486     // if NZCV is used, do not fold.
487     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
488       return 0;
489     // fall-through to SUBXrr and SUBWrr.
490     LLVM_FALLTHROUGH;
491   case AArch64::SUBXrr:
492   case AArch64::SUBWrr: {
493     // neg x -> csneg, represented as sub dst, xzr, src.
494     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
495     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
496       return 0;
497     SrcOpNum = 2;
498     Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
499     break;
500   }
501   default:
502     return 0;
503   }
504   assert(Opc && SrcOpNum && "Missing parameters");
505 
506   if (NewVReg)
507     *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
508   return Opc;
509 }
510 
511 bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
512                                        ArrayRef<MachineOperand> Cond,
513                                        Register DstReg, Register TrueReg,
514                                        Register FalseReg, int &CondCycles,
515                                        int &TrueCycles,
516                                        int &FalseCycles) const {
517   // Check register classes.
518   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
519   const TargetRegisterClass *RC =
520       RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
521   if (!RC)
522     return false;
523 
524   // Also need to check the dest regclass, in case we're trying to optimize
525   // something like:
526   // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
527   if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
528     return false;
529 
530   // Expanding cbz/tbz requires an extra cycle of latency on the condition.
531   unsigned ExtraCondLat = Cond.size() != 1;
532 
533   // GPRs are handled by csel.
534   // FIXME: Fold in x+1, -x, and ~x when applicable.
535   if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
536       AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
537     // Single-cycle csel, csinc, csinv, and csneg.
538     CondCycles = 1 + ExtraCondLat;
539     TrueCycles = FalseCycles = 1;
540     if (canFoldIntoCSel(MRI, TrueReg))
541       TrueCycles = 0;
542     else if (canFoldIntoCSel(MRI, FalseReg))
543       FalseCycles = 0;
544     return true;
545   }
546 
547   // Scalar floating point is handled by fcsel.
548   // FIXME: Form fabs, fmin, and fmax when applicable.
549   if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
550       AArch64::FPR32RegClass.hasSubClassEq(RC)) {
551     CondCycles = 5 + ExtraCondLat;
552     TrueCycles = FalseCycles = 2;
553     return true;
554   }
555 
556   // Can't do vectors.
557   return false;
558 }
559 
560 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
561                                     MachineBasicBlock::iterator I,
562                                     const DebugLoc &DL, Register DstReg,
563                                     ArrayRef<MachineOperand> Cond,
564                                     Register TrueReg, Register FalseReg) const {
565   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
566 
567   // Parse the condition code, see parseCondBranch() above.
568   AArch64CC::CondCode CC;
569   switch (Cond.size()) {
570   default:
571     llvm_unreachable("Unknown condition opcode in Cond");
572   case 1: // b.cc
573     CC = AArch64CC::CondCode(Cond[0].getImm());
574     break;
575   case 3: { // cbz/cbnz
576     // We must insert a compare against 0.
577     bool Is64Bit;
578     switch (Cond[1].getImm()) {
579     default:
580       llvm_unreachable("Unknown branch opcode in Cond");
581     case AArch64::CBZW:
582       Is64Bit = false;
583       CC = AArch64CC::EQ;
584       break;
585     case AArch64::CBZX:
586       Is64Bit = true;
587       CC = AArch64CC::EQ;
588       break;
589     case AArch64::CBNZW:
590       Is64Bit = false;
591       CC = AArch64CC::NE;
592       break;
593     case AArch64::CBNZX:
594       Is64Bit = true;
595       CC = AArch64CC::NE;
596       break;
597     }
598     Register SrcReg = Cond[2].getReg();
599     if (Is64Bit) {
600       // cmp reg, #0 is actually subs xzr, reg, #0.
601       MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
602       BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
603           .addReg(SrcReg)
604           .addImm(0)
605           .addImm(0);
606     } else {
607       MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
608       BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
609           .addReg(SrcReg)
610           .addImm(0)
611           .addImm(0);
612     }
613     break;
614   }
615   case 4: { // tbz/tbnz
616     // We must insert a tst instruction.
617     switch (Cond[1].getImm()) {
618     default:
619       llvm_unreachable("Unknown branch opcode in Cond");
620     case AArch64::TBZW:
621     case AArch64::TBZX:
622       CC = AArch64CC::EQ;
623       break;
624     case AArch64::TBNZW:
625     case AArch64::TBNZX:
626       CC = AArch64CC::NE;
627       break;
628     }
629     // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
630     if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
631       BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
632           .addReg(Cond[2].getReg())
633           .addImm(
634               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
635     else
636       BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
637           .addReg(Cond[2].getReg())
638           .addImm(
639               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
640     break;
641   }
642   }
643 
644   unsigned Opc = 0;
645   const TargetRegisterClass *RC = nullptr;
646   bool TryFold = false;
647   if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
648     RC = &AArch64::GPR64RegClass;
649     Opc = AArch64::CSELXr;
650     TryFold = true;
651   } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
652     RC = &AArch64::GPR32RegClass;
653     Opc = AArch64::CSELWr;
654     TryFold = true;
655   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
656     RC = &AArch64::FPR64RegClass;
657     Opc = AArch64::FCSELDrrr;
658   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
659     RC = &AArch64::FPR32RegClass;
660     Opc = AArch64::FCSELSrrr;
661   }
662   assert(RC && "Unsupported regclass");
663 
664   // Try folding simple instructions into the csel.
665   if (TryFold) {
666     unsigned NewVReg = 0;
667     unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
668     if (FoldedOpc) {
669       // The folded opcodes csinc, csinc and csneg apply the operation to
670       // FalseReg, so we need to invert the condition.
671       CC = AArch64CC::getInvertedCondCode(CC);
672       TrueReg = FalseReg;
673     } else
674       FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
675 
676     // Fold the operation. Leave any dead instructions for DCE to clean up.
677     if (FoldedOpc) {
678       FalseReg = NewVReg;
679       Opc = FoldedOpc;
680       // The extends the live range of NewVReg.
681       MRI.clearKillFlags(NewVReg);
682     }
683   }
684 
685   // Pull all virtual register into the appropriate class.
686   MRI.constrainRegClass(TrueReg, RC);
687   MRI.constrainRegClass(FalseReg, RC);
688 
689   // Insert the csel.
690   BuildMI(MBB, I, DL, get(Opc), DstReg)
691       .addReg(TrueReg)
692       .addReg(FalseReg)
693       .addImm(CC);
694 }
695 
696 /// Returns true if a MOVi32imm or MOVi64imm can be expanded to an  ORRxx.
697 static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
698   uint64_t Imm = MI.getOperand(1).getImm();
699   uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
700   uint64_t Encoding;
701   return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
702 }
703 
704 // FIXME: this implementation should be micro-architecture dependent, so a
705 // micro-architecture target hook should be introduced here in future.
706 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
707   if (!Subtarget.hasCustomCheapAsMoveHandling())
708     return MI.isAsCheapAsAMove();
709 
710   const unsigned Opcode = MI.getOpcode();
711 
712   // Firstly, check cases gated by features.
713 
714   if (Subtarget.hasZeroCycleZeroingFP()) {
715     if (Opcode == AArch64::FMOVH0 ||
716         Opcode == AArch64::FMOVS0 ||
717         Opcode == AArch64::FMOVD0)
718       return true;
719   }
720 
721   if (Subtarget.hasZeroCycleZeroingGP()) {
722     if (Opcode == TargetOpcode::COPY &&
723         (MI.getOperand(1).getReg() == AArch64::WZR ||
724          MI.getOperand(1).getReg() == AArch64::XZR))
725       return true;
726   }
727 
728   // Secondly, check cases specific to sub-targets.
729 
730   if (Subtarget.hasExynosCheapAsMoveHandling()) {
731     if (isExynosCheapAsMove(MI))
732       return true;
733 
734     return MI.isAsCheapAsAMove();
735   }
736 
737   // Finally, check generic cases.
738 
739   switch (Opcode) {
740   default:
741     return false;
742 
743   // add/sub on register without shift
744   case AArch64::ADDWri:
745   case AArch64::ADDXri:
746   case AArch64::SUBWri:
747   case AArch64::SUBXri:
748     return (MI.getOperand(3).getImm() == 0);
749 
750   // logical ops on immediate
751   case AArch64::ANDWri:
752   case AArch64::ANDXri:
753   case AArch64::EORWri:
754   case AArch64::EORXri:
755   case AArch64::ORRWri:
756   case AArch64::ORRXri:
757     return true;
758 
759   // logical ops on register without shift
760   case AArch64::ANDWrr:
761   case AArch64::ANDXrr:
762   case AArch64::BICWrr:
763   case AArch64::BICXrr:
764   case AArch64::EONWrr:
765   case AArch64::EONXrr:
766   case AArch64::EORWrr:
767   case AArch64::EORXrr:
768   case AArch64::ORNWrr:
769   case AArch64::ORNXrr:
770   case AArch64::ORRWrr:
771   case AArch64::ORRXrr:
772     return true;
773 
774   // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
775   // ORRXri, it is as cheap as MOV
776   case AArch64::MOVi32imm:
777     return canBeExpandedToORR(MI, 32);
778   case AArch64::MOVi64imm:
779     return canBeExpandedToORR(MI, 64);
780   }
781 
782   llvm_unreachable("Unknown opcode to check as cheap as a move!");
783 }
784 
785 bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
786   switch (MI.getOpcode()) {
787   default:
788     return false;
789 
790   case AArch64::ADDWrs:
791   case AArch64::ADDXrs:
792   case AArch64::ADDSWrs:
793   case AArch64::ADDSXrs: {
794     unsigned Imm = MI.getOperand(3).getImm();
795     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
796     if (ShiftVal == 0)
797       return true;
798     return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
799   }
800 
801   case AArch64::ADDWrx:
802   case AArch64::ADDXrx:
803   case AArch64::ADDXrx64:
804   case AArch64::ADDSWrx:
805   case AArch64::ADDSXrx:
806   case AArch64::ADDSXrx64: {
807     unsigned Imm = MI.getOperand(3).getImm();
808     switch (AArch64_AM::getArithExtendType(Imm)) {
809     default:
810       return false;
811     case AArch64_AM::UXTB:
812     case AArch64_AM::UXTH:
813     case AArch64_AM::UXTW:
814     case AArch64_AM::UXTX:
815       return AArch64_AM::getArithShiftValue(Imm) <= 4;
816     }
817   }
818 
819   case AArch64::SUBWrs:
820   case AArch64::SUBSWrs: {
821     unsigned Imm = MI.getOperand(3).getImm();
822     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
823     return ShiftVal == 0 ||
824            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
825   }
826 
827   case AArch64::SUBXrs:
828   case AArch64::SUBSXrs: {
829     unsigned Imm = MI.getOperand(3).getImm();
830     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
831     return ShiftVal == 0 ||
832            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
833   }
834 
835   case AArch64::SUBWrx:
836   case AArch64::SUBXrx:
837   case AArch64::SUBXrx64:
838   case AArch64::SUBSWrx:
839   case AArch64::SUBSXrx:
840   case AArch64::SUBSXrx64: {
841     unsigned Imm = MI.getOperand(3).getImm();
842     switch (AArch64_AM::getArithExtendType(Imm)) {
843     default:
844       return false;
845     case AArch64_AM::UXTB:
846     case AArch64_AM::UXTH:
847     case AArch64_AM::UXTW:
848     case AArch64_AM::UXTX:
849       return AArch64_AM::getArithShiftValue(Imm) == 0;
850     }
851   }
852 
853   case AArch64::LDRBBroW:
854   case AArch64::LDRBBroX:
855   case AArch64::LDRBroW:
856   case AArch64::LDRBroX:
857   case AArch64::LDRDroW:
858   case AArch64::LDRDroX:
859   case AArch64::LDRHHroW:
860   case AArch64::LDRHHroX:
861   case AArch64::LDRHroW:
862   case AArch64::LDRHroX:
863   case AArch64::LDRQroW:
864   case AArch64::LDRQroX:
865   case AArch64::LDRSBWroW:
866   case AArch64::LDRSBWroX:
867   case AArch64::LDRSBXroW:
868   case AArch64::LDRSBXroX:
869   case AArch64::LDRSHWroW:
870   case AArch64::LDRSHWroX:
871   case AArch64::LDRSHXroW:
872   case AArch64::LDRSHXroX:
873   case AArch64::LDRSWroW:
874   case AArch64::LDRSWroX:
875   case AArch64::LDRSroW:
876   case AArch64::LDRSroX:
877   case AArch64::LDRWroW:
878   case AArch64::LDRWroX:
879   case AArch64::LDRXroW:
880   case AArch64::LDRXroX:
881   case AArch64::PRFMroW:
882   case AArch64::PRFMroX:
883   case AArch64::STRBBroW:
884   case AArch64::STRBBroX:
885   case AArch64::STRBroW:
886   case AArch64::STRBroX:
887   case AArch64::STRDroW:
888   case AArch64::STRDroX:
889   case AArch64::STRHHroW:
890   case AArch64::STRHHroX:
891   case AArch64::STRHroW:
892   case AArch64::STRHroX:
893   case AArch64::STRQroW:
894   case AArch64::STRQroX:
895   case AArch64::STRSroW:
896   case AArch64::STRSroX:
897   case AArch64::STRWroW:
898   case AArch64::STRWroX:
899   case AArch64::STRXroW:
900   case AArch64::STRXroX: {
901     unsigned IsSigned = MI.getOperand(3).getImm();
902     return !IsSigned;
903   }
904   }
905 }
906 
907 bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
908   unsigned Opc = MI.getOpcode();
909   switch (Opc) {
910     default:
911       return false;
912     case AArch64::SEH_StackAlloc:
913     case AArch64::SEH_SaveFPLR:
914     case AArch64::SEH_SaveFPLR_X:
915     case AArch64::SEH_SaveReg:
916     case AArch64::SEH_SaveReg_X:
917     case AArch64::SEH_SaveRegP:
918     case AArch64::SEH_SaveRegP_X:
919     case AArch64::SEH_SaveFReg:
920     case AArch64::SEH_SaveFReg_X:
921     case AArch64::SEH_SaveFRegP:
922     case AArch64::SEH_SaveFRegP_X:
923     case AArch64::SEH_SetFP:
924     case AArch64::SEH_AddFP:
925     case AArch64::SEH_Nop:
926     case AArch64::SEH_PrologEnd:
927     case AArch64::SEH_EpilogStart:
928     case AArch64::SEH_EpilogEnd:
929       return true;
930   }
931 }
932 
933 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
934                                              Register &SrcReg, Register &DstReg,
935                                              unsigned &SubIdx) const {
936   switch (MI.getOpcode()) {
937   default:
938     return false;
939   case AArch64::SBFMXri: // aka sxtw
940   case AArch64::UBFMXri: // aka uxtw
941     // Check for the 32 -> 64 bit extension case, these instructions can do
942     // much more.
943     if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
944       return false;
945     // This is a signed or unsigned 32 -> 64 bit extension.
946     SrcReg = MI.getOperand(1).getReg();
947     DstReg = MI.getOperand(0).getReg();
948     SubIdx = AArch64::sub_32;
949     return true;
950   }
951 }
952 
953 bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
954     const MachineInstr &MIa, const MachineInstr &MIb) const {
955   const TargetRegisterInfo *TRI = &getRegisterInfo();
956   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
957   int64_t OffsetA = 0, OffsetB = 0;
958   unsigned WidthA = 0, WidthB = 0;
959   bool OffsetAIsScalable = false, OffsetBIsScalable = false;
960 
961   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
962   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
963 
964   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
965       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
966     return false;
967 
968   // Retrieve the base, offset from the base and width. Width
969   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8).  If
970   // base are identical, and the offset of a lower memory access +
971   // the width doesn't overlap the offset of a higher memory access,
972   // then the memory accesses are different.
973   // If OffsetAIsScalable and OffsetBIsScalable are both true, they
974   // are assumed to have the same scale (vscale).
975   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
976                                    WidthA, TRI) &&
977       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
978                                    WidthB, TRI)) {
979     if (BaseOpA->isIdenticalTo(*BaseOpB) &&
980         OffsetAIsScalable == OffsetBIsScalable) {
981       int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
982       int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
983       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
984       if (LowOffset + LowWidth <= HighOffset)
985         return true;
986     }
987   }
988   return false;
989 }
990 
991 bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
992                                             const MachineBasicBlock *MBB,
993                                             const MachineFunction &MF) const {
994   if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
995     return true;
996   switch (MI.getOpcode()) {
997   case AArch64::HINT:
998     // CSDB hints are scheduling barriers.
999     if (MI.getOperand(0).getImm() == 0x14)
1000       return true;
1001     break;
1002   case AArch64::DSB:
1003   case AArch64::ISB:
1004     // DSB and ISB also are scheduling barriers.
1005     return true;
1006   default:;
1007   }
1008   return isSEHInstruction(MI);
1009 }
1010 
1011 /// analyzeCompare - For a comparison instruction, return the source registers
1012 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1013 /// Return true if the comparison instruction can be analyzed.
1014 bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1015                                       Register &SrcReg2, int &CmpMask,
1016                                       int &CmpValue) const {
1017   // The first operand can be a frame index where we'd normally expect a
1018   // register.
1019   assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1020   if (!MI.getOperand(1).isReg())
1021     return false;
1022 
1023   switch (MI.getOpcode()) {
1024   default:
1025     break;
1026   case AArch64::SUBSWrr:
1027   case AArch64::SUBSWrs:
1028   case AArch64::SUBSWrx:
1029   case AArch64::SUBSXrr:
1030   case AArch64::SUBSXrs:
1031   case AArch64::SUBSXrx:
1032   case AArch64::ADDSWrr:
1033   case AArch64::ADDSWrs:
1034   case AArch64::ADDSWrx:
1035   case AArch64::ADDSXrr:
1036   case AArch64::ADDSXrs:
1037   case AArch64::ADDSXrx:
1038     // Replace SUBSWrr with SUBWrr if NZCV is not used.
1039     SrcReg = MI.getOperand(1).getReg();
1040     SrcReg2 = MI.getOperand(2).getReg();
1041     CmpMask = ~0;
1042     CmpValue = 0;
1043     return true;
1044   case AArch64::SUBSWri:
1045   case AArch64::ADDSWri:
1046   case AArch64::SUBSXri:
1047   case AArch64::ADDSXri:
1048     SrcReg = MI.getOperand(1).getReg();
1049     SrcReg2 = 0;
1050     CmpMask = ~0;
1051     // FIXME: In order to convert CmpValue to 0 or 1
1052     CmpValue = MI.getOperand(2).getImm() != 0;
1053     return true;
1054   case AArch64::ANDSWri:
1055   case AArch64::ANDSXri:
1056     // ANDS does not use the same encoding scheme as the others xxxS
1057     // instructions.
1058     SrcReg = MI.getOperand(1).getReg();
1059     SrcReg2 = 0;
1060     CmpMask = ~0;
1061     // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
1062     // while the type of CmpValue is int. When converting uint64_t to int,
1063     // the high 32 bits of uint64_t will be lost.
1064     // In fact it causes a bug in spec2006-483.xalancbmk
1065     // CmpValue is only used to compare with zero in OptimizeCompareInstr
1066     CmpValue = AArch64_AM::decodeLogicalImmediate(
1067                    MI.getOperand(2).getImm(),
1068                    MI.getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0;
1069     return true;
1070   }
1071 
1072   return false;
1073 }
1074 
1075 static bool UpdateOperandRegClass(MachineInstr &Instr) {
1076   MachineBasicBlock *MBB = Instr.getParent();
1077   assert(MBB && "Can't get MachineBasicBlock here");
1078   MachineFunction *MF = MBB->getParent();
1079   assert(MF && "Can't get MachineFunction here");
1080   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1081   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1082   MachineRegisterInfo *MRI = &MF->getRegInfo();
1083 
1084   for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1085        ++OpIdx) {
1086     MachineOperand &MO = Instr.getOperand(OpIdx);
1087     const TargetRegisterClass *OpRegCstraints =
1088         Instr.getRegClassConstraint(OpIdx, TII, TRI);
1089 
1090     // If there's no constraint, there's nothing to do.
1091     if (!OpRegCstraints)
1092       continue;
1093     // If the operand is a frame index, there's nothing to do here.
1094     // A frame index operand will resolve correctly during PEI.
1095     if (MO.isFI())
1096       continue;
1097 
1098     assert(MO.isReg() &&
1099            "Operand has register constraints without being a register!");
1100 
1101     Register Reg = MO.getReg();
1102     if (Register::isPhysicalRegister(Reg)) {
1103       if (!OpRegCstraints->contains(Reg))
1104         return false;
1105     } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
1106                !MRI->constrainRegClass(Reg, OpRegCstraints))
1107       return false;
1108   }
1109 
1110   return true;
1111 }
1112 
1113 /// Return the opcode that does not set flags when possible - otherwise
1114 /// return the original opcode. The caller is responsible to do the actual
1115 /// substitution and legality checking.
1116 static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1117   // Don't convert all compare instructions, because for some the zero register
1118   // encoding becomes the sp register.
1119   bool MIDefinesZeroReg = false;
1120   if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR))
1121     MIDefinesZeroReg = true;
1122 
1123   switch (MI.getOpcode()) {
1124   default:
1125     return MI.getOpcode();
1126   case AArch64::ADDSWrr:
1127     return AArch64::ADDWrr;
1128   case AArch64::ADDSWri:
1129     return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1130   case AArch64::ADDSWrs:
1131     return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1132   case AArch64::ADDSWrx:
1133     return AArch64::ADDWrx;
1134   case AArch64::ADDSXrr:
1135     return AArch64::ADDXrr;
1136   case AArch64::ADDSXri:
1137     return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1138   case AArch64::ADDSXrs:
1139     return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1140   case AArch64::ADDSXrx:
1141     return AArch64::ADDXrx;
1142   case AArch64::SUBSWrr:
1143     return AArch64::SUBWrr;
1144   case AArch64::SUBSWri:
1145     return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1146   case AArch64::SUBSWrs:
1147     return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1148   case AArch64::SUBSWrx:
1149     return AArch64::SUBWrx;
1150   case AArch64::SUBSXrr:
1151     return AArch64::SUBXrr;
1152   case AArch64::SUBSXri:
1153     return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1154   case AArch64::SUBSXrs:
1155     return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1156   case AArch64::SUBSXrx:
1157     return AArch64::SUBXrx;
1158   }
1159 }
1160 
1161 enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1162 
1163 /// True when condition flags are accessed (either by writing or reading)
1164 /// on the instruction trace starting at From and ending at To.
1165 ///
1166 /// Note: If From and To are from different blocks it's assumed CC are accessed
1167 ///       on the path.
1168 static bool areCFlagsAccessedBetweenInstrs(
1169     MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1170     const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1171   // Early exit if To is at the beginning of the BB.
1172   if (To == To->getParent()->begin())
1173     return true;
1174 
1175   // Check whether the instructions are in the same basic block
1176   // If not, assume the condition flags might get modified somewhere.
1177   if (To->getParent() != From->getParent())
1178     return true;
1179 
1180   // From must be above To.
1181   assert(std::find_if(++To.getReverse(), To->getParent()->rend(),
1182                       [From](MachineInstr &MI) {
1183                         return MI.getIterator() == From;
1184                       }) != To->getParent()->rend());
1185 
1186   // We iterate backward starting at \p To until we hit \p From.
1187   for (const MachineInstr &Instr :
1188        instructionsWithoutDebug(++To.getReverse(), From.getReverse())) {
1189     if (((AccessToCheck & AK_Write) &&
1190          Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
1191         ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
1192       return true;
1193   }
1194   return false;
1195 }
1196 
1197 /// Try to optimize a compare instruction. A compare instruction is an
1198 /// instruction which produces AArch64::NZCV. It can be truly compare
1199 /// instruction
1200 /// when there are no uses of its destination register.
1201 ///
1202 /// The following steps are tried in order:
1203 /// 1. Convert CmpInstr into an unconditional version.
1204 /// 2. Remove CmpInstr if above there is an instruction producing a needed
1205 ///    condition code or an instruction which can be converted into such an
1206 ///    instruction.
1207 ///    Only comparison with zero is supported.
1208 bool AArch64InstrInfo::optimizeCompareInstr(
1209     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
1210     int CmpValue, const MachineRegisterInfo *MRI) const {
1211   assert(CmpInstr.getParent());
1212   assert(MRI);
1213 
1214   // Replace SUBSWrr with SUBWrr if NZCV is not used.
1215   int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true);
1216   if (DeadNZCVIdx != -1) {
1217     if (CmpInstr.definesRegister(AArch64::WZR) ||
1218         CmpInstr.definesRegister(AArch64::XZR)) {
1219       CmpInstr.eraseFromParent();
1220       return true;
1221     }
1222     unsigned Opc = CmpInstr.getOpcode();
1223     unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
1224     if (NewOpc == Opc)
1225       return false;
1226     const MCInstrDesc &MCID = get(NewOpc);
1227     CmpInstr.setDesc(MCID);
1228     CmpInstr.RemoveOperand(DeadNZCVIdx);
1229     bool succeeded = UpdateOperandRegClass(CmpInstr);
1230     (void)succeeded;
1231     assert(succeeded && "Some operands reg class are incompatible!");
1232     return true;
1233   }
1234 
1235   // Continue only if we have a "ri" where immediate is zero.
1236   // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
1237   // function.
1238   assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
1239   if (CmpValue != 0 || SrcReg2 != 0)
1240     return false;
1241 
1242   // CmpInstr is a Compare instruction if destination register is not used.
1243   if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
1244     return false;
1245 
1246   return substituteCmpToZero(CmpInstr, SrcReg, MRI);
1247 }
1248 
1249 /// Get opcode of S version of Instr.
1250 /// If Instr is S version its opcode is returned.
1251 /// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
1252 /// or we are not interested in it.
1253 static unsigned sForm(MachineInstr &Instr) {
1254   switch (Instr.getOpcode()) {
1255   default:
1256     return AArch64::INSTRUCTION_LIST_END;
1257 
1258   case AArch64::ADDSWrr:
1259   case AArch64::ADDSWri:
1260   case AArch64::ADDSXrr:
1261   case AArch64::ADDSXri:
1262   case AArch64::SUBSWrr:
1263   case AArch64::SUBSWri:
1264   case AArch64::SUBSXrr:
1265   case AArch64::SUBSXri:
1266     return Instr.getOpcode();
1267 
1268   case AArch64::ADDWrr:
1269     return AArch64::ADDSWrr;
1270   case AArch64::ADDWri:
1271     return AArch64::ADDSWri;
1272   case AArch64::ADDXrr:
1273     return AArch64::ADDSXrr;
1274   case AArch64::ADDXri:
1275     return AArch64::ADDSXri;
1276   case AArch64::ADCWr:
1277     return AArch64::ADCSWr;
1278   case AArch64::ADCXr:
1279     return AArch64::ADCSXr;
1280   case AArch64::SUBWrr:
1281     return AArch64::SUBSWrr;
1282   case AArch64::SUBWri:
1283     return AArch64::SUBSWri;
1284   case AArch64::SUBXrr:
1285     return AArch64::SUBSXrr;
1286   case AArch64::SUBXri:
1287     return AArch64::SUBSXri;
1288   case AArch64::SBCWr:
1289     return AArch64::SBCSWr;
1290   case AArch64::SBCXr:
1291     return AArch64::SBCSXr;
1292   case AArch64::ANDWri:
1293     return AArch64::ANDSWri;
1294   case AArch64::ANDXri:
1295     return AArch64::ANDSXri;
1296   }
1297 }
1298 
1299 /// Check if AArch64::NZCV should be alive in successors of MBB.
1300 static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) {
1301   for (auto *BB : MBB->successors())
1302     if (BB->isLiveIn(AArch64::NZCV))
1303       return true;
1304   return false;
1305 }
1306 
1307 namespace {
1308 
1309 struct UsedNZCV {
1310   bool N = false;
1311   bool Z = false;
1312   bool C = false;
1313   bool V = false;
1314 
1315   UsedNZCV() = default;
1316 
1317   UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
1318     this->N |= UsedFlags.N;
1319     this->Z |= UsedFlags.Z;
1320     this->C |= UsedFlags.C;
1321     this->V |= UsedFlags.V;
1322     return *this;
1323   }
1324 };
1325 
1326 } // end anonymous namespace
1327 
1328 /// Find a condition code used by the instruction.
1329 /// Returns AArch64CC::Invalid if either the instruction does not use condition
1330 /// codes or we don't optimize CmpInstr in the presence of such instructions.
1331 static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
1332   switch (Instr.getOpcode()) {
1333   default:
1334     return AArch64CC::Invalid;
1335 
1336   case AArch64::Bcc: {
1337     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1338     assert(Idx >= 2);
1339     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 2).getImm());
1340   }
1341 
1342   case AArch64::CSINVWr:
1343   case AArch64::CSINVXr:
1344   case AArch64::CSINCWr:
1345   case AArch64::CSINCXr:
1346   case AArch64::CSELWr:
1347   case AArch64::CSELXr:
1348   case AArch64::CSNEGWr:
1349   case AArch64::CSNEGXr:
1350   case AArch64::FCSELSrrr:
1351   case AArch64::FCSELDrrr: {
1352     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1353     assert(Idx >= 1);
1354     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 1).getImm());
1355   }
1356   }
1357 }
1358 
1359 static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
1360   assert(CC != AArch64CC::Invalid);
1361   UsedNZCV UsedFlags;
1362   switch (CC) {
1363   default:
1364     break;
1365 
1366   case AArch64CC::EQ: // Z set
1367   case AArch64CC::NE: // Z clear
1368     UsedFlags.Z = true;
1369     break;
1370 
1371   case AArch64CC::HI: // Z clear and C set
1372   case AArch64CC::LS: // Z set   or  C clear
1373     UsedFlags.Z = true;
1374     LLVM_FALLTHROUGH;
1375   case AArch64CC::HS: // C set
1376   case AArch64CC::LO: // C clear
1377     UsedFlags.C = true;
1378     break;
1379 
1380   case AArch64CC::MI: // N set
1381   case AArch64CC::PL: // N clear
1382     UsedFlags.N = true;
1383     break;
1384 
1385   case AArch64CC::VS: // V set
1386   case AArch64CC::VC: // V clear
1387     UsedFlags.V = true;
1388     break;
1389 
1390   case AArch64CC::GT: // Z clear, N and V the same
1391   case AArch64CC::LE: // Z set,   N and V differ
1392     UsedFlags.Z = true;
1393     LLVM_FALLTHROUGH;
1394   case AArch64CC::GE: // N and V the same
1395   case AArch64CC::LT: // N and V differ
1396     UsedFlags.N = true;
1397     UsedFlags.V = true;
1398     break;
1399   }
1400   return UsedFlags;
1401 }
1402 
1403 static bool isADDSRegImm(unsigned Opcode) {
1404   return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1405 }
1406 
1407 static bool isSUBSRegImm(unsigned Opcode) {
1408   return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1409 }
1410 
1411 /// Check if CmpInstr can be substituted by MI.
1412 ///
1413 /// CmpInstr can be substituted:
1414 /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1415 /// - and, MI and CmpInstr are from the same MachineBB
1416 /// - and, condition flags are not alive in successors of the CmpInstr parent
1417 /// - and, if MI opcode is the S form there must be no defs of flags between
1418 ///        MI and CmpInstr
1419 ///        or if MI opcode is not the S form there must be neither defs of flags
1420 ///        nor uses of flags between MI and CmpInstr.
1421 /// - and  C/V flags are not used after CmpInstr
1422 static bool canInstrSubstituteCmpInstr(MachineInstr *MI, MachineInstr *CmpInstr,
1423                                        const TargetRegisterInfo *TRI) {
1424   assert(MI);
1425   assert(sForm(*MI) != AArch64::INSTRUCTION_LIST_END);
1426   assert(CmpInstr);
1427 
1428   const unsigned CmpOpcode = CmpInstr->getOpcode();
1429   if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
1430     return false;
1431 
1432   if (MI->getParent() != CmpInstr->getParent())
1433     return false;
1434 
1435   if (areCFlagsAliveInSuccessors(CmpInstr->getParent()))
1436     return false;
1437 
1438   AccessKind AccessToCheck = AK_Write;
1439   if (sForm(*MI) != MI->getOpcode())
1440     AccessToCheck = AK_All;
1441   if (areCFlagsAccessedBetweenInstrs(MI, CmpInstr, TRI, AccessToCheck))
1442     return false;
1443 
1444   UsedNZCV NZCVUsedAfterCmp;
1445   for (const MachineInstr &Instr :
1446        instructionsWithoutDebug(std::next(CmpInstr->getIterator()),
1447                                 CmpInstr->getParent()->instr_end())) {
1448     if (Instr.readsRegister(AArch64::NZCV, TRI)) {
1449       AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
1450       if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
1451         return false;
1452       NZCVUsedAfterCmp |= getUsedNZCV(CC);
1453     }
1454 
1455     if (Instr.modifiesRegister(AArch64::NZCV, TRI))
1456       break;
1457   }
1458 
1459   return !NZCVUsedAfterCmp.C && !NZCVUsedAfterCmp.V;
1460 }
1461 
1462 /// Substitute an instruction comparing to zero with another instruction
1463 /// which produces needed condition flags.
1464 ///
1465 /// Return true on success.
1466 bool AArch64InstrInfo::substituteCmpToZero(
1467     MachineInstr &CmpInstr, unsigned SrcReg,
1468     const MachineRegisterInfo *MRI) const {
1469   assert(MRI);
1470   // Get the unique definition of SrcReg.
1471   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1472   if (!MI)
1473     return false;
1474 
1475   const TargetRegisterInfo *TRI = &getRegisterInfo();
1476 
1477   unsigned NewOpc = sForm(*MI);
1478   if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1479     return false;
1480 
1481   if (!canInstrSubstituteCmpInstr(MI, &CmpInstr, TRI))
1482     return false;
1483 
1484   // Update the instruction to set NZCV.
1485   MI->setDesc(get(NewOpc));
1486   CmpInstr.eraseFromParent();
1487   bool succeeded = UpdateOperandRegClass(*MI);
1488   (void)succeeded;
1489   assert(succeeded && "Some operands reg class are incompatible!");
1490   MI->addRegisterDefined(AArch64::NZCV, TRI);
1491   return true;
1492 }
1493 
1494 bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1495   if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1496       MI.getOpcode() != AArch64::CATCHRET)
1497     return false;
1498 
1499   MachineBasicBlock &MBB = *MI.getParent();
1500   auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1501   auto TRI = Subtarget.getRegisterInfo();
1502   DebugLoc DL = MI.getDebugLoc();
1503 
1504   if (MI.getOpcode() == AArch64::CATCHRET) {
1505     // Skip to the first instruction before the epilog.
1506     const TargetInstrInfo *TII =
1507       MBB.getParent()->getSubtarget().getInstrInfo();
1508     MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
1509     auto MBBI = MachineBasicBlock::iterator(MI);
1510     MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
1511     while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
1512            FirstEpilogSEH != MBB.begin())
1513       FirstEpilogSEH = std::prev(FirstEpilogSEH);
1514     if (FirstEpilogSEH != MBB.begin())
1515       FirstEpilogSEH = std::next(FirstEpilogSEH);
1516     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
1517         .addReg(AArch64::X0, RegState::Define)
1518         .addMBB(TargetMBB);
1519     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
1520         .addReg(AArch64::X0, RegState::Define)
1521         .addReg(AArch64::X0)
1522         .addMBB(TargetMBB)
1523         .addImm(0);
1524     return true;
1525   }
1526 
1527   Register Reg = MI.getOperand(0).getReg();
1528   const GlobalValue *GV =
1529       cast<GlobalValue>((*MI.memoperands_begin())->getValue());
1530   const TargetMachine &TM = MBB.getParent()->getTarget();
1531   unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
1532   const unsigned char MO_NC = AArch64II::MO_NC;
1533 
1534   if ((OpFlags & AArch64II::MO_GOT) != 0) {
1535     BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
1536         .addGlobalAddress(GV, 0, OpFlags);
1537     if (Subtarget.isTargetILP32()) {
1538       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1539       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1540           .addDef(Reg32, RegState::Dead)
1541           .addUse(Reg, RegState::Kill)
1542           .addImm(0)
1543           .addMemOperand(*MI.memoperands_begin())
1544           .addDef(Reg, RegState::Implicit);
1545     } else {
1546       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1547           .addReg(Reg, RegState::Kill)
1548           .addImm(0)
1549           .addMemOperand(*MI.memoperands_begin());
1550     }
1551   } else if (TM.getCodeModel() == CodeModel::Large) {
1552     assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?");
1553     BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
1554         .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
1555         .addImm(0);
1556     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1557         .addReg(Reg, RegState::Kill)
1558         .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
1559         .addImm(16);
1560     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1561         .addReg(Reg, RegState::Kill)
1562         .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
1563         .addImm(32);
1564     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1565         .addReg(Reg, RegState::Kill)
1566         .addGlobalAddress(GV, 0, AArch64II::MO_G3)
1567         .addImm(48);
1568     BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1569         .addReg(Reg, RegState::Kill)
1570         .addImm(0)
1571         .addMemOperand(*MI.memoperands_begin());
1572   } else if (TM.getCodeModel() == CodeModel::Tiny) {
1573     BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg)
1574         .addGlobalAddress(GV, 0, OpFlags);
1575   } else {
1576     BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
1577         .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
1578     unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
1579     if (Subtarget.isTargetILP32()) {
1580       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1581       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1582           .addDef(Reg32, RegState::Dead)
1583           .addUse(Reg, RegState::Kill)
1584           .addGlobalAddress(GV, 0, LoFlags)
1585           .addMemOperand(*MI.memoperands_begin())
1586           .addDef(Reg, RegState::Implicit);
1587     } else {
1588       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1589           .addReg(Reg, RegState::Kill)
1590           .addGlobalAddress(GV, 0, LoFlags)
1591           .addMemOperand(*MI.memoperands_begin());
1592     }
1593   }
1594 
1595   MBB.erase(MI);
1596 
1597   return true;
1598 }
1599 
1600 // Return true if this instruction simply sets its single destination register
1601 // to zero. This is equivalent to a register rename of the zero-register.
1602 bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
1603   switch (MI.getOpcode()) {
1604   default:
1605     break;
1606   case AArch64::MOVZWi:
1607   case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1608     if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
1609       assert(MI.getDesc().getNumOperands() == 3 &&
1610              MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1611       return true;
1612     }
1613     break;
1614   case AArch64::ANDWri: // and Rd, Rzr, #imm
1615     return MI.getOperand(1).getReg() == AArch64::WZR;
1616   case AArch64::ANDXri:
1617     return MI.getOperand(1).getReg() == AArch64::XZR;
1618   case TargetOpcode::COPY:
1619     return MI.getOperand(1).getReg() == AArch64::WZR;
1620   }
1621   return false;
1622 }
1623 
1624 // Return true if this instruction simply renames a general register without
1625 // modifying bits.
1626 bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
1627   switch (MI.getOpcode()) {
1628   default:
1629     break;
1630   case TargetOpcode::COPY: {
1631     // GPR32 copies will by lowered to ORRXrs
1632     Register DstReg = MI.getOperand(0).getReg();
1633     return (AArch64::GPR32RegClass.contains(DstReg) ||
1634             AArch64::GPR64RegClass.contains(DstReg));
1635   }
1636   case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1637     if (MI.getOperand(1).getReg() == AArch64::XZR) {
1638       assert(MI.getDesc().getNumOperands() == 4 &&
1639              MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1640       return true;
1641     }
1642     break;
1643   case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1644     if (MI.getOperand(2).getImm() == 0) {
1645       assert(MI.getDesc().getNumOperands() == 4 &&
1646              MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1647       return true;
1648     }
1649     break;
1650   }
1651   return false;
1652 }
1653 
1654 // Return true if this instruction simply renames a general register without
1655 // modifying bits.
1656 bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
1657   switch (MI.getOpcode()) {
1658   default:
1659     break;
1660   case TargetOpcode::COPY: {
1661     // FPR64 copies will by lowered to ORR.16b
1662     Register DstReg = MI.getOperand(0).getReg();
1663     return (AArch64::FPR64RegClass.contains(DstReg) ||
1664             AArch64::FPR128RegClass.contains(DstReg));
1665   }
1666   case AArch64::ORRv16i8:
1667     if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
1668       assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
1669              "invalid ORRv16i8 operands");
1670       return true;
1671     }
1672     break;
1673   }
1674   return false;
1675 }
1676 
1677 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1678                                                int &FrameIndex) const {
1679   switch (MI.getOpcode()) {
1680   default:
1681     break;
1682   case AArch64::LDRWui:
1683   case AArch64::LDRXui:
1684   case AArch64::LDRBui:
1685   case AArch64::LDRHui:
1686   case AArch64::LDRSui:
1687   case AArch64::LDRDui:
1688   case AArch64::LDRQui:
1689     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1690         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1691       FrameIndex = MI.getOperand(1).getIndex();
1692       return MI.getOperand(0).getReg();
1693     }
1694     break;
1695   }
1696 
1697   return 0;
1698 }
1699 
1700 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1701                                               int &FrameIndex) const {
1702   switch (MI.getOpcode()) {
1703   default:
1704     break;
1705   case AArch64::STRWui:
1706   case AArch64::STRXui:
1707   case AArch64::STRBui:
1708   case AArch64::STRHui:
1709   case AArch64::STRSui:
1710   case AArch64::STRDui:
1711   case AArch64::STRQui:
1712   case AArch64::LDR_PXI:
1713   case AArch64::STR_PXI:
1714     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1715         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1716       FrameIndex = MI.getOperand(1).getIndex();
1717       return MI.getOperand(0).getReg();
1718     }
1719     break;
1720   }
1721   return 0;
1722 }
1723 
1724 /// Check all MachineMemOperands for a hint to suppress pairing.
1725 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
1726   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1727     return MMO->getFlags() & MOSuppressPair;
1728   });
1729 }
1730 
1731 /// Set a flag on the first MachineMemOperand to suppress pairing.
1732 void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
1733   if (MI.memoperands_empty())
1734     return;
1735   (*MI.memoperands_begin())->setFlags(MOSuppressPair);
1736 }
1737 
1738 /// Check all MachineMemOperands for a hint that the load/store is strided.
1739 bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
1740   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1741     return MMO->getFlags() & MOStridedAccess;
1742   });
1743 }
1744 
1745 bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) {
1746   switch (Opc) {
1747   default:
1748     return false;
1749   case AArch64::STURSi:
1750   case AArch64::STURDi:
1751   case AArch64::STURQi:
1752   case AArch64::STURBBi:
1753   case AArch64::STURHHi:
1754   case AArch64::STURWi:
1755   case AArch64::STURXi:
1756   case AArch64::LDURSi:
1757   case AArch64::LDURDi:
1758   case AArch64::LDURQi:
1759   case AArch64::LDURWi:
1760   case AArch64::LDURXi:
1761   case AArch64::LDURSWi:
1762   case AArch64::LDURHHi:
1763   case AArch64::LDURBBi:
1764   case AArch64::LDURSBWi:
1765   case AArch64::LDURSHWi:
1766     return true;
1767   }
1768 }
1769 
1770 Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
1771   switch (Opc) {
1772   default: return {};
1773   case AArch64::PRFMui: return AArch64::PRFUMi;
1774   case AArch64::LDRXui: return AArch64::LDURXi;
1775   case AArch64::LDRWui: return AArch64::LDURWi;
1776   case AArch64::LDRBui: return AArch64::LDURBi;
1777   case AArch64::LDRHui: return AArch64::LDURHi;
1778   case AArch64::LDRSui: return AArch64::LDURSi;
1779   case AArch64::LDRDui: return AArch64::LDURDi;
1780   case AArch64::LDRQui: return AArch64::LDURQi;
1781   case AArch64::LDRBBui: return AArch64::LDURBBi;
1782   case AArch64::LDRHHui: return AArch64::LDURHHi;
1783   case AArch64::LDRSBXui: return AArch64::LDURSBXi;
1784   case AArch64::LDRSBWui: return AArch64::LDURSBWi;
1785   case AArch64::LDRSHXui: return AArch64::LDURSHXi;
1786   case AArch64::LDRSHWui: return AArch64::LDURSHWi;
1787   case AArch64::LDRSWui: return AArch64::LDURSWi;
1788   case AArch64::STRXui: return AArch64::STURXi;
1789   case AArch64::STRWui: return AArch64::STURWi;
1790   case AArch64::STRBui: return AArch64::STURBi;
1791   case AArch64::STRHui: return AArch64::STURHi;
1792   case AArch64::STRSui: return AArch64::STURSi;
1793   case AArch64::STRDui: return AArch64::STURDi;
1794   case AArch64::STRQui: return AArch64::STURQi;
1795   case AArch64::STRBBui: return AArch64::STURBBi;
1796   case AArch64::STRHHui: return AArch64::STURHHi;
1797   }
1798 }
1799 
1800 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
1801   switch (Opc) {
1802   default:
1803     return 2;
1804   case AArch64::LDPXi:
1805   case AArch64::LDPDi:
1806   case AArch64::STPXi:
1807   case AArch64::STPDi:
1808   case AArch64::LDNPXi:
1809   case AArch64::LDNPDi:
1810   case AArch64::STNPXi:
1811   case AArch64::STNPDi:
1812   case AArch64::LDPQi:
1813   case AArch64::STPQi:
1814   case AArch64::LDNPQi:
1815   case AArch64::STNPQi:
1816   case AArch64::LDPWi:
1817   case AArch64::LDPSi:
1818   case AArch64::STPWi:
1819   case AArch64::STPSi:
1820   case AArch64::LDNPWi:
1821   case AArch64::LDNPSi:
1822   case AArch64::STNPWi:
1823   case AArch64::STNPSi:
1824   case AArch64::LDG:
1825   case AArch64::STGPi:
1826   case AArch64::LD1B_IMM:
1827   case AArch64::LD1H_IMM:
1828   case AArch64::LD1W_IMM:
1829   case AArch64::LD1D_IMM:
1830   case AArch64::ST1B_IMM:
1831   case AArch64::ST1H_IMM:
1832   case AArch64::ST1W_IMM:
1833   case AArch64::ST1D_IMM:
1834   case AArch64::LD1B_H_IMM:
1835   case AArch64::LD1SB_H_IMM:
1836   case AArch64::LD1H_S_IMM:
1837   case AArch64::LD1SH_S_IMM:
1838   case AArch64::LD1W_D_IMM:
1839   case AArch64::LD1SW_D_IMM:
1840   case AArch64::ST1B_H_IMM:
1841   case AArch64::ST1H_S_IMM:
1842   case AArch64::ST1W_D_IMM:
1843   case AArch64::LD1B_S_IMM:
1844   case AArch64::LD1SB_S_IMM:
1845   case AArch64::LD1H_D_IMM:
1846   case AArch64::LD1SH_D_IMM:
1847   case AArch64::ST1B_S_IMM:
1848   case AArch64::ST1H_D_IMM:
1849   case AArch64::LD1B_D_IMM:
1850   case AArch64::LD1SB_D_IMM:
1851   case AArch64::ST1B_D_IMM:
1852     return 3;
1853   case AArch64::ADDG:
1854   case AArch64::STGOffset:
1855   case AArch64::LDR_PXI:
1856   case AArch64::STR_PXI:
1857     return 2;
1858   }
1859 }
1860 
1861 bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
1862   switch (MI.getOpcode()) {
1863   default:
1864     return false;
1865   // Scaled instructions.
1866   case AArch64::STRSui:
1867   case AArch64::STRDui:
1868   case AArch64::STRQui:
1869   case AArch64::STRXui:
1870   case AArch64::STRWui:
1871   case AArch64::LDRSui:
1872   case AArch64::LDRDui:
1873   case AArch64::LDRQui:
1874   case AArch64::LDRXui:
1875   case AArch64::LDRWui:
1876   case AArch64::LDRSWui:
1877   // Unscaled instructions.
1878   case AArch64::STURSi:
1879   case AArch64::STURDi:
1880   case AArch64::STURQi:
1881   case AArch64::STURWi:
1882   case AArch64::STURXi:
1883   case AArch64::LDURSi:
1884   case AArch64::LDURDi:
1885   case AArch64::LDURQi:
1886   case AArch64::LDURWi:
1887   case AArch64::LDURXi:
1888   case AArch64::LDURSWi:
1889     return true;
1890   }
1891 }
1892 
1893 unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc,
1894                                                    bool &Is64Bit) {
1895   switch (Opc) {
1896   default:
1897     llvm_unreachable("Opcode has no flag setting equivalent!");
1898   // 32-bit cases:
1899   case AArch64::ADDWri:
1900     Is64Bit = false;
1901     return AArch64::ADDSWri;
1902   case AArch64::ADDWrr:
1903     Is64Bit = false;
1904     return AArch64::ADDSWrr;
1905   case AArch64::ADDWrs:
1906     Is64Bit = false;
1907     return AArch64::ADDSWrs;
1908   case AArch64::ADDWrx:
1909     Is64Bit = false;
1910     return AArch64::ADDSWrx;
1911   case AArch64::ANDWri:
1912     Is64Bit = false;
1913     return AArch64::ANDSWri;
1914   case AArch64::ANDWrr:
1915     Is64Bit = false;
1916     return AArch64::ANDSWrr;
1917   case AArch64::ANDWrs:
1918     Is64Bit = false;
1919     return AArch64::ANDSWrs;
1920   case AArch64::BICWrr:
1921     Is64Bit = false;
1922     return AArch64::BICSWrr;
1923   case AArch64::BICWrs:
1924     Is64Bit = false;
1925     return AArch64::BICSWrs;
1926   case AArch64::SUBWri:
1927     Is64Bit = false;
1928     return AArch64::SUBSWri;
1929   case AArch64::SUBWrr:
1930     Is64Bit = false;
1931     return AArch64::SUBSWrr;
1932   case AArch64::SUBWrs:
1933     Is64Bit = false;
1934     return AArch64::SUBSWrs;
1935   case AArch64::SUBWrx:
1936     Is64Bit = false;
1937     return AArch64::SUBSWrx;
1938   // 64-bit cases:
1939   case AArch64::ADDXri:
1940     Is64Bit = true;
1941     return AArch64::ADDSXri;
1942   case AArch64::ADDXrr:
1943     Is64Bit = true;
1944     return AArch64::ADDSXrr;
1945   case AArch64::ADDXrs:
1946     Is64Bit = true;
1947     return AArch64::ADDSXrs;
1948   case AArch64::ADDXrx:
1949     Is64Bit = true;
1950     return AArch64::ADDSXrx;
1951   case AArch64::ANDXri:
1952     Is64Bit = true;
1953     return AArch64::ANDSXri;
1954   case AArch64::ANDXrr:
1955     Is64Bit = true;
1956     return AArch64::ANDSXrr;
1957   case AArch64::ANDXrs:
1958     Is64Bit = true;
1959     return AArch64::ANDSXrs;
1960   case AArch64::BICXrr:
1961     Is64Bit = true;
1962     return AArch64::BICSXrr;
1963   case AArch64::BICXrs:
1964     Is64Bit = true;
1965     return AArch64::BICSXrs;
1966   case AArch64::SUBXri:
1967     Is64Bit = true;
1968     return AArch64::SUBSXri;
1969   case AArch64::SUBXrr:
1970     Is64Bit = true;
1971     return AArch64::SUBSXrr;
1972   case AArch64::SUBXrs:
1973     Is64Bit = true;
1974     return AArch64::SUBSXrs;
1975   case AArch64::SUBXrx:
1976     Is64Bit = true;
1977     return AArch64::SUBSXrx;
1978   }
1979 }
1980 
1981 // Is this a candidate for ld/st merging or pairing?  For example, we don't
1982 // touch volatiles or load/stores that have a hint to avoid pair formation.
1983 bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
1984   // If this is a volatile load/store, don't mess with it.
1985   if (MI.hasOrderedMemoryRef())
1986     return false;
1987 
1988   // Make sure this is a reg/fi+imm (as opposed to an address reloc).
1989   assert((MI.getOperand(1).isReg() || MI.getOperand(1).isFI()) &&
1990          "Expected a reg or frame index operand.");
1991   if (!MI.getOperand(2).isImm())
1992     return false;
1993 
1994   // Can't merge/pair if the instruction modifies the base register.
1995   // e.g., ldr x0, [x0]
1996   // This case will never occur with an FI base.
1997   if (MI.getOperand(1).isReg()) {
1998     Register BaseReg = MI.getOperand(1).getReg();
1999     const TargetRegisterInfo *TRI = &getRegisterInfo();
2000     if (MI.modifiesRegister(BaseReg, TRI))
2001       return false;
2002   }
2003 
2004   // Check if this load/store has a hint to avoid pair formation.
2005   // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
2006   if (isLdStPairSuppressed(MI))
2007     return false;
2008 
2009   // Do not pair any callee-save store/reload instructions in the
2010   // prologue/epilogue if the CFI information encoded the operations as separate
2011   // instructions, as that will cause the size of the actual prologue to mismatch
2012   // with the prologue size recorded in the Windows CFI.
2013   const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo();
2014   bool NeedsWinCFI = MAI->usesWindowsCFI() &&
2015                      MI.getMF()->getFunction().needsUnwindTableEntry();
2016   if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
2017                       MI.getFlag(MachineInstr::FrameDestroy)))
2018     return false;
2019 
2020   // On some CPUs quad load/store pairs are slower than two single load/stores.
2021   if (Subtarget.isPaired128Slow()) {
2022     switch (MI.getOpcode()) {
2023     default:
2024       break;
2025     case AArch64::LDURQi:
2026     case AArch64::STURQi:
2027     case AArch64::LDRQui:
2028     case AArch64::STRQui:
2029       return false;
2030     }
2031   }
2032 
2033   return true;
2034 }
2035 
2036 bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
2037     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2038     int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2039     const TargetRegisterInfo *TRI) const {
2040   if (!LdSt.mayLoadOrStore())
2041     return false;
2042 
2043   const MachineOperand *BaseOp;
2044   if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
2045                                     Width, TRI))
2046     return false;
2047   BaseOps.push_back(BaseOp);
2048   return true;
2049 }
2050 
2051 bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
2052     const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
2053     bool &OffsetIsScalable, unsigned &Width,
2054     const TargetRegisterInfo *TRI) const {
2055   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2056   // Handle only loads/stores with base register followed by immediate offset.
2057   if (LdSt.getNumExplicitOperands() == 3) {
2058     // Non-paired instruction (e.g., ldr x1, [x0, #8]).
2059     if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2060         !LdSt.getOperand(2).isImm())
2061       return false;
2062   } else if (LdSt.getNumExplicitOperands() == 4) {
2063     // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
2064     if (!LdSt.getOperand(1).isReg() ||
2065         (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
2066         !LdSt.getOperand(3).isImm())
2067       return false;
2068   } else
2069     return false;
2070 
2071   // Get the scaling factor for the instruction and set the width for the
2072   // instruction.
2073   TypeSize Scale(0U, false);
2074   int64_t Dummy1, Dummy2;
2075 
2076   // If this returns false, then it's an instruction we don't want to handle.
2077   if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
2078     return false;
2079 
2080   // Compute the offset. Offset is calculated as the immediate operand
2081   // multiplied by the scaling factor. Unscaled instructions have scaling factor
2082   // set to 1.
2083   if (LdSt.getNumExplicitOperands() == 3) {
2084     BaseOp = &LdSt.getOperand(1);
2085     Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinSize();
2086   } else {
2087     assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
2088     BaseOp = &LdSt.getOperand(2);
2089     Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinSize();
2090   }
2091   OffsetIsScalable = Scale.isScalable();
2092 
2093   if (!BaseOp->isReg() && !BaseOp->isFI())
2094     return false;
2095 
2096   return true;
2097 }
2098 
2099 MachineOperand &
2100 AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
2101   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2102   MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
2103   assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
2104   return OfsOp;
2105 }
2106 
2107 bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
2108                                     unsigned &Width, int64_t &MinOffset,
2109                                     int64_t &MaxOffset) {
2110   const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
2111   switch (Opcode) {
2112   // Not a memory operation or something we want to handle.
2113   default:
2114     Scale = TypeSize::Fixed(0);
2115     Width = 0;
2116     MinOffset = MaxOffset = 0;
2117     return false;
2118   case AArch64::STRWpost:
2119   case AArch64::LDRWpost:
2120     Width = 32;
2121     Scale = TypeSize::Fixed(4);
2122     MinOffset = -256;
2123     MaxOffset = 255;
2124     break;
2125   case AArch64::LDURQi:
2126   case AArch64::STURQi:
2127     Width = 16;
2128     Scale = TypeSize::Fixed(1);
2129     MinOffset = -256;
2130     MaxOffset = 255;
2131     break;
2132   case AArch64::PRFUMi:
2133   case AArch64::LDURXi:
2134   case AArch64::LDURDi:
2135   case AArch64::STURXi:
2136   case AArch64::STURDi:
2137     Width = 8;
2138     Scale = TypeSize::Fixed(1);
2139     MinOffset = -256;
2140     MaxOffset = 255;
2141     break;
2142   case AArch64::LDURWi:
2143   case AArch64::LDURSi:
2144   case AArch64::LDURSWi:
2145   case AArch64::STURWi:
2146   case AArch64::STURSi:
2147     Width = 4;
2148     Scale = TypeSize::Fixed(1);
2149     MinOffset = -256;
2150     MaxOffset = 255;
2151     break;
2152   case AArch64::LDURHi:
2153   case AArch64::LDURHHi:
2154   case AArch64::LDURSHXi:
2155   case AArch64::LDURSHWi:
2156   case AArch64::STURHi:
2157   case AArch64::STURHHi:
2158     Width = 2;
2159     Scale = TypeSize::Fixed(1);
2160     MinOffset = -256;
2161     MaxOffset = 255;
2162     break;
2163   case AArch64::LDURBi:
2164   case AArch64::LDURBBi:
2165   case AArch64::LDURSBXi:
2166   case AArch64::LDURSBWi:
2167   case AArch64::STURBi:
2168   case AArch64::STURBBi:
2169     Width = 1;
2170     Scale = TypeSize::Fixed(1);
2171     MinOffset = -256;
2172     MaxOffset = 255;
2173     break;
2174   case AArch64::LDPQi:
2175   case AArch64::LDNPQi:
2176   case AArch64::STPQi:
2177   case AArch64::STNPQi:
2178     Scale = TypeSize::Fixed(16);
2179     Width = 32;
2180     MinOffset = -64;
2181     MaxOffset = 63;
2182     break;
2183   case AArch64::LDRQui:
2184   case AArch64::STRQui:
2185     Scale = TypeSize::Fixed(16);
2186     Width = 16;
2187     MinOffset = 0;
2188     MaxOffset = 4095;
2189     break;
2190   case AArch64::LDPXi:
2191   case AArch64::LDPDi:
2192   case AArch64::LDNPXi:
2193   case AArch64::LDNPDi:
2194   case AArch64::STPXi:
2195   case AArch64::STPDi:
2196   case AArch64::STNPXi:
2197   case AArch64::STNPDi:
2198     Scale = TypeSize::Fixed(8);
2199     Width = 16;
2200     MinOffset = -64;
2201     MaxOffset = 63;
2202     break;
2203   case AArch64::PRFMui:
2204   case AArch64::LDRXui:
2205   case AArch64::LDRDui:
2206   case AArch64::STRXui:
2207   case AArch64::STRDui:
2208     Scale = TypeSize::Fixed(8);
2209     Width = 8;
2210     MinOffset = 0;
2211     MaxOffset = 4095;
2212     break;
2213   case AArch64::LDPWi:
2214   case AArch64::LDPSi:
2215   case AArch64::LDNPWi:
2216   case AArch64::LDNPSi:
2217   case AArch64::STPWi:
2218   case AArch64::STPSi:
2219   case AArch64::STNPWi:
2220   case AArch64::STNPSi:
2221     Scale = TypeSize::Fixed(4);
2222     Width = 8;
2223     MinOffset = -64;
2224     MaxOffset = 63;
2225     break;
2226   case AArch64::LDRWui:
2227   case AArch64::LDRSui:
2228   case AArch64::LDRSWui:
2229   case AArch64::STRWui:
2230   case AArch64::STRSui:
2231     Scale = TypeSize::Fixed(4);
2232     Width = 4;
2233     MinOffset = 0;
2234     MaxOffset = 4095;
2235     break;
2236   case AArch64::LDRHui:
2237   case AArch64::LDRHHui:
2238   case AArch64::LDRSHWui:
2239   case AArch64::LDRSHXui:
2240   case AArch64::STRHui:
2241   case AArch64::STRHHui:
2242     Scale = TypeSize::Fixed(2);
2243     Width = 2;
2244     MinOffset = 0;
2245     MaxOffset = 4095;
2246     break;
2247   case AArch64::LDRBui:
2248   case AArch64::LDRBBui:
2249   case AArch64::LDRSBWui:
2250   case AArch64::LDRSBXui:
2251   case AArch64::STRBui:
2252   case AArch64::STRBBui:
2253     Scale = TypeSize::Fixed(1);
2254     Width = 1;
2255     MinOffset = 0;
2256     MaxOffset = 4095;
2257     break;
2258   case AArch64::ADDG:
2259     Scale = TypeSize::Fixed(16);
2260     Width = 0;
2261     MinOffset = 0;
2262     MaxOffset = 63;
2263     break;
2264   case AArch64::TAGPstack:
2265     Scale = TypeSize::Fixed(16);
2266     Width = 0;
2267     // TAGP with a negative offset turns into SUBP, which has a maximum offset
2268     // of 63 (not 64!).
2269     MinOffset = -63;
2270     MaxOffset = 63;
2271     break;
2272   case AArch64::LDG:
2273   case AArch64::STGOffset:
2274   case AArch64::STZGOffset:
2275     Scale = TypeSize::Fixed(16);
2276     Width = 16;
2277     MinOffset = -256;
2278     MaxOffset = 255;
2279     break;
2280   case AArch64::STR_ZZZZXI:
2281   case AArch64::LDR_ZZZZXI:
2282     Scale = TypeSize::Scalable(16);
2283     Width = SVEMaxBytesPerVector * 4;
2284     MinOffset = -256;
2285     MaxOffset = 252;
2286     break;
2287   case AArch64::STR_ZZZXI:
2288   case AArch64::LDR_ZZZXI:
2289     Scale = TypeSize::Scalable(16);
2290     Width = SVEMaxBytesPerVector * 3;
2291     MinOffset = -256;
2292     MaxOffset = 253;
2293     break;
2294   case AArch64::STR_ZZXI:
2295   case AArch64::LDR_ZZXI:
2296     Scale = TypeSize::Scalable(16);
2297     Width = SVEMaxBytesPerVector * 2;
2298     MinOffset = -256;
2299     MaxOffset = 254;
2300     break;
2301   case AArch64::LDR_PXI:
2302   case AArch64::STR_PXI:
2303     Scale = TypeSize::Scalable(2);
2304     Width = SVEMaxBytesPerVector / 8;
2305     MinOffset = -256;
2306     MaxOffset = 255;
2307     break;
2308   case AArch64::LDR_ZXI:
2309   case AArch64::STR_ZXI:
2310     Scale = TypeSize::Scalable(16);
2311     Width = SVEMaxBytesPerVector;
2312     MinOffset = -256;
2313     MaxOffset = 255;
2314     break;
2315   case AArch64::LD1B_IMM:
2316   case AArch64::LD1H_IMM:
2317   case AArch64::LD1W_IMM:
2318   case AArch64::LD1D_IMM:
2319   case AArch64::ST1B_IMM:
2320   case AArch64::ST1H_IMM:
2321   case AArch64::ST1W_IMM:
2322   case AArch64::ST1D_IMM:
2323     // A full vectors worth of data
2324     // Width = mbytes * elements
2325     Scale = TypeSize::Scalable(16);
2326     Width = SVEMaxBytesPerVector;
2327     MinOffset = -8;
2328     MaxOffset = 7;
2329     break;
2330   case AArch64::LD1B_H_IMM:
2331   case AArch64::LD1SB_H_IMM:
2332   case AArch64::LD1H_S_IMM:
2333   case AArch64::LD1SH_S_IMM:
2334   case AArch64::LD1W_D_IMM:
2335   case AArch64::LD1SW_D_IMM:
2336   case AArch64::ST1B_H_IMM:
2337   case AArch64::ST1H_S_IMM:
2338   case AArch64::ST1W_D_IMM:
2339     // A half vector worth of data
2340     // Width = mbytes * elements
2341     Scale = TypeSize::Scalable(8);
2342     Width = SVEMaxBytesPerVector / 2;
2343     MinOffset = -8;
2344     MaxOffset = 7;
2345     break;
2346   case AArch64::LD1B_S_IMM:
2347   case AArch64::LD1SB_S_IMM:
2348   case AArch64::LD1H_D_IMM:
2349   case AArch64::LD1SH_D_IMM:
2350   case AArch64::ST1B_S_IMM:
2351   case AArch64::ST1H_D_IMM:
2352     // A quarter vector worth of data
2353     // Width = mbytes * elements
2354     Scale = TypeSize::Scalable(4);
2355     Width = SVEMaxBytesPerVector / 4;
2356     MinOffset = -8;
2357     MaxOffset = 7;
2358     break;
2359   case AArch64::LD1B_D_IMM:
2360   case AArch64::LD1SB_D_IMM:
2361   case AArch64::ST1B_D_IMM:
2362     // A eighth vector worth of data
2363     // Width = mbytes * elements
2364     Scale = TypeSize::Scalable(2);
2365     Width = SVEMaxBytesPerVector / 8;
2366     MinOffset = -8;
2367     MaxOffset = 7;
2368     break;
2369   case AArch64::ST2GOffset:
2370   case AArch64::STZ2GOffset:
2371     Scale = TypeSize::Fixed(16);
2372     Width = 32;
2373     MinOffset = -256;
2374     MaxOffset = 255;
2375     break;
2376   case AArch64::STGPi:
2377     Scale = TypeSize::Fixed(16);
2378     Width = 16;
2379     MinOffset = -64;
2380     MaxOffset = 63;
2381     break;
2382   }
2383 
2384   return true;
2385 }
2386 
2387 // Scaling factor for unscaled load or store.
2388 int AArch64InstrInfo::getMemScale(unsigned Opc) {
2389   switch (Opc) {
2390   default:
2391     llvm_unreachable("Opcode has unknown scale!");
2392   case AArch64::LDRBBui:
2393   case AArch64::LDURBBi:
2394   case AArch64::LDRSBWui:
2395   case AArch64::LDURSBWi:
2396   case AArch64::STRBBui:
2397   case AArch64::STURBBi:
2398     return 1;
2399   case AArch64::LDRHHui:
2400   case AArch64::LDURHHi:
2401   case AArch64::LDRSHWui:
2402   case AArch64::LDURSHWi:
2403   case AArch64::STRHHui:
2404   case AArch64::STURHHi:
2405     return 2;
2406   case AArch64::LDRSui:
2407   case AArch64::LDURSi:
2408   case AArch64::LDRSWui:
2409   case AArch64::LDURSWi:
2410   case AArch64::LDRWui:
2411   case AArch64::LDURWi:
2412   case AArch64::STRSui:
2413   case AArch64::STURSi:
2414   case AArch64::STRWui:
2415   case AArch64::STURWi:
2416   case AArch64::LDPSi:
2417   case AArch64::LDPSWi:
2418   case AArch64::LDPWi:
2419   case AArch64::STPSi:
2420   case AArch64::STPWi:
2421     return 4;
2422   case AArch64::LDRDui:
2423   case AArch64::LDURDi:
2424   case AArch64::LDRXui:
2425   case AArch64::LDURXi:
2426   case AArch64::STRDui:
2427   case AArch64::STURDi:
2428   case AArch64::STRXui:
2429   case AArch64::STURXi:
2430   case AArch64::LDPDi:
2431   case AArch64::LDPXi:
2432   case AArch64::STPDi:
2433   case AArch64::STPXi:
2434     return 8;
2435   case AArch64::LDRQui:
2436   case AArch64::LDURQi:
2437   case AArch64::STRQui:
2438   case AArch64::STURQi:
2439   case AArch64::LDPQi:
2440   case AArch64::STPQi:
2441   case AArch64::STGOffset:
2442   case AArch64::STZGOffset:
2443   case AArch64::ST2GOffset:
2444   case AArch64::STZ2GOffset:
2445   case AArch64::STGPi:
2446     return 16;
2447   }
2448 }
2449 
2450 // Scale the unscaled offsets.  Returns false if the unscaled offset can't be
2451 // scaled.
2452 static bool scaleOffset(unsigned Opc, int64_t &Offset) {
2453   int Scale = AArch64InstrInfo::getMemScale(Opc);
2454 
2455   // If the byte-offset isn't a multiple of the stride, we can't scale this
2456   // offset.
2457   if (Offset % Scale != 0)
2458     return false;
2459 
2460   // Convert the byte-offset used by unscaled into an "element" offset used
2461   // by the scaled pair load/store instructions.
2462   Offset /= Scale;
2463   return true;
2464 }
2465 
2466 static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
2467   if (FirstOpc == SecondOpc)
2468     return true;
2469   // We can also pair sign-ext and zero-ext instructions.
2470   switch (FirstOpc) {
2471   default:
2472     return false;
2473   case AArch64::LDRWui:
2474   case AArch64::LDURWi:
2475     return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
2476   case AArch64::LDRSWui:
2477   case AArch64::LDURSWi:
2478     return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
2479   }
2480   // These instructions can't be paired based on their opcodes.
2481   return false;
2482 }
2483 
2484 static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
2485                             int64_t Offset1, unsigned Opcode1, int FI2,
2486                             int64_t Offset2, unsigned Opcode2) {
2487   // Accesses through fixed stack object frame indices may access a different
2488   // fixed stack slot. Check that the object offsets + offsets match.
2489   if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
2490     int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
2491     int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
2492     assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
2493     // Convert to scaled object offsets.
2494     int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
2495     if (ObjectOffset1 % Scale1 != 0)
2496       return false;
2497     ObjectOffset1 /= Scale1;
2498     int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
2499     if (ObjectOffset2 % Scale2 != 0)
2500       return false;
2501     ObjectOffset2 /= Scale2;
2502     ObjectOffset1 += Offset1;
2503     ObjectOffset2 += Offset2;
2504     return ObjectOffset1 + 1 == ObjectOffset2;
2505   }
2506 
2507   return FI1 == FI2;
2508 }
2509 
2510 /// Detect opportunities for ldp/stp formation.
2511 ///
2512 /// Only called for LdSt for which getMemOperandWithOffset returns true.
2513 bool AArch64InstrInfo::shouldClusterMemOps(
2514     ArrayRef<const MachineOperand *> BaseOps1,
2515     ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2516     unsigned NumBytes) const {
2517   assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2518   const MachineOperand &BaseOp1 = *BaseOps1.front();
2519   const MachineOperand &BaseOp2 = *BaseOps2.front();
2520   const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2521   const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2522   if (BaseOp1.getType() != BaseOp2.getType())
2523     return false;
2524 
2525   assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2526          "Only base registers and frame indices are supported.");
2527 
2528   // Check for both base regs and base FI.
2529   if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
2530     return false;
2531 
2532   // Only cluster up to a single pair.
2533   if (NumLoads > 2)
2534     return false;
2535 
2536   if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
2537     return false;
2538 
2539   // Can we pair these instructions based on their opcodes?
2540   unsigned FirstOpc = FirstLdSt.getOpcode();
2541   unsigned SecondOpc = SecondLdSt.getOpcode();
2542   if (!canPairLdStOpc(FirstOpc, SecondOpc))
2543     return false;
2544 
2545   // Can't merge volatiles or load/stores that have a hint to avoid pair
2546   // formation, for example.
2547   if (!isCandidateToMergeOrPair(FirstLdSt) ||
2548       !isCandidateToMergeOrPair(SecondLdSt))
2549     return false;
2550 
2551   // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
2552   int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
2553   if (isUnscaledLdSt(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
2554     return false;
2555 
2556   int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
2557   if (isUnscaledLdSt(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
2558     return false;
2559 
2560   // Pairwise instructions have a 7-bit signed offset field.
2561   if (Offset1 > 63 || Offset1 < -64)
2562     return false;
2563 
2564   // The caller should already have ordered First/SecondLdSt by offset.
2565   // Note: except for non-equal frame index bases
2566   if (BaseOp1.isFI()) {
2567     assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
2568            "Caller should have ordered offsets.");
2569 
2570     const MachineFrameInfo &MFI =
2571         FirstLdSt.getParent()->getParent()->getFrameInfo();
2572     return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
2573                            BaseOp2.getIndex(), Offset2, SecondOpc);
2574   }
2575 
2576   assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2577 
2578   return Offset1 + 1 == Offset2;
2579 }
2580 
2581 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
2582                                             unsigned Reg, unsigned SubIdx,
2583                                             unsigned State,
2584                                             const TargetRegisterInfo *TRI) {
2585   if (!SubIdx)
2586     return MIB.addReg(Reg, State);
2587 
2588   if (Register::isPhysicalRegister(Reg))
2589     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
2590   return MIB.addReg(Reg, State, SubIdx);
2591 }
2592 
2593 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
2594                                         unsigned NumRegs) {
2595   // We really want the positive remainder mod 32 here, that happens to be
2596   // easily obtainable with a mask.
2597   return ((DestReg - SrcReg) & 0x1f) < NumRegs;
2598 }
2599 
2600 void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
2601                                         MachineBasicBlock::iterator I,
2602                                         const DebugLoc &DL, MCRegister DestReg,
2603                                         MCRegister SrcReg, bool KillSrc,
2604                                         unsigned Opcode,
2605                                         ArrayRef<unsigned> Indices) const {
2606   assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
2607   const TargetRegisterInfo *TRI = &getRegisterInfo();
2608   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2609   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2610   unsigned NumRegs = Indices.size();
2611 
2612   int SubReg = 0, End = NumRegs, Incr = 1;
2613   if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
2614     SubReg = NumRegs - 1;
2615     End = -1;
2616     Incr = -1;
2617   }
2618 
2619   for (; SubReg != End; SubReg += Incr) {
2620     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2621     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2622     AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
2623     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2624   }
2625 }
2626 
2627 void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
2628                                        MachineBasicBlock::iterator I,
2629                                        DebugLoc DL, unsigned DestReg,
2630                                        unsigned SrcReg, bool KillSrc,
2631                                        unsigned Opcode, unsigned ZeroReg,
2632                                        llvm::ArrayRef<unsigned> Indices) const {
2633   const TargetRegisterInfo *TRI = &getRegisterInfo();
2634   unsigned NumRegs = Indices.size();
2635 
2636 #ifndef NDEBUG
2637   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2638   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2639   assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
2640          "GPR reg sequences should not be able to overlap");
2641 #endif
2642 
2643   for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
2644     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2645     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2646     MIB.addReg(ZeroReg);
2647     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2648     MIB.addImm(0);
2649   }
2650 }
2651 
2652 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
2653                                    MachineBasicBlock::iterator I,
2654                                    const DebugLoc &DL, MCRegister DestReg,
2655                                    MCRegister SrcReg, bool KillSrc) const {
2656   if (AArch64::GPR32spRegClass.contains(DestReg) &&
2657       (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
2658     const TargetRegisterInfo *TRI = &getRegisterInfo();
2659 
2660     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
2661       // If either operand is WSP, expand to ADD #0.
2662       if (Subtarget.hasZeroCycleRegMove()) {
2663         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
2664         MCRegister DestRegX = TRI->getMatchingSuperReg(
2665             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2666         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2667             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2668         // This instruction is reading and writing X registers.  This may upset
2669         // the register scavenger and machine verifier, so we need to indicate
2670         // that we are reading an undefined value from SrcRegX, but a proper
2671         // value from SrcReg.
2672         BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
2673             .addReg(SrcRegX, RegState::Undef)
2674             .addImm(0)
2675             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2676             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2677       } else {
2678         BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
2679             .addReg(SrcReg, getKillRegState(KillSrc))
2680             .addImm(0)
2681             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2682       }
2683     } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
2684       BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
2685           .addImm(0)
2686           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2687     } else {
2688       if (Subtarget.hasZeroCycleRegMove()) {
2689         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
2690         MCRegister DestRegX = TRI->getMatchingSuperReg(
2691             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2692         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2693             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2694         // This instruction is reading and writing X registers.  This may upset
2695         // the register scavenger and machine verifier, so we need to indicate
2696         // that we are reading an undefined value from SrcRegX, but a proper
2697         // value from SrcReg.
2698         BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
2699             .addReg(AArch64::XZR)
2700             .addReg(SrcRegX, RegState::Undef)
2701             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2702       } else {
2703         // Otherwise, expand to ORR WZR.
2704         BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
2705             .addReg(AArch64::WZR)
2706             .addReg(SrcReg, getKillRegState(KillSrc));
2707       }
2708     }
2709     return;
2710   }
2711 
2712   // Copy a Predicate register by ORRing with itself.
2713   if (AArch64::PPRRegClass.contains(DestReg) &&
2714       AArch64::PPRRegClass.contains(SrcReg)) {
2715     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2716     BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
2717       .addReg(SrcReg) // Pg
2718       .addReg(SrcReg)
2719       .addReg(SrcReg, getKillRegState(KillSrc));
2720     return;
2721   }
2722 
2723   // Copy a Z register by ORRing with itself.
2724   if (AArch64::ZPRRegClass.contains(DestReg) &&
2725       AArch64::ZPRRegClass.contains(SrcReg)) {
2726     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2727     BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
2728       .addReg(SrcReg)
2729       .addReg(SrcReg, getKillRegState(KillSrc));
2730     return;
2731   }
2732 
2733   if (AArch64::GPR64spRegClass.contains(DestReg) &&
2734       (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
2735     if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
2736       // If either operand is SP, expand to ADD #0.
2737       BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
2738           .addReg(SrcReg, getKillRegState(KillSrc))
2739           .addImm(0)
2740           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2741     } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
2742       BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
2743           .addImm(0)
2744           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2745     } else {
2746       // Otherwise, expand to ORR XZR.
2747       BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
2748           .addReg(AArch64::XZR)
2749           .addReg(SrcReg, getKillRegState(KillSrc));
2750     }
2751     return;
2752   }
2753 
2754   // Copy a DDDD register quad by copying the individual sub-registers.
2755   if (AArch64::DDDDRegClass.contains(DestReg) &&
2756       AArch64::DDDDRegClass.contains(SrcReg)) {
2757     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2758                                        AArch64::dsub2, AArch64::dsub3};
2759     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2760                      Indices);
2761     return;
2762   }
2763 
2764   // Copy a DDD register triple by copying the individual sub-registers.
2765   if (AArch64::DDDRegClass.contains(DestReg) &&
2766       AArch64::DDDRegClass.contains(SrcReg)) {
2767     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2768                                        AArch64::dsub2};
2769     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2770                      Indices);
2771     return;
2772   }
2773 
2774   // Copy a DD register pair by copying the individual sub-registers.
2775   if (AArch64::DDRegClass.contains(DestReg) &&
2776       AArch64::DDRegClass.contains(SrcReg)) {
2777     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
2778     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2779                      Indices);
2780     return;
2781   }
2782 
2783   // Copy a QQQQ register quad by copying the individual sub-registers.
2784   if (AArch64::QQQQRegClass.contains(DestReg) &&
2785       AArch64::QQQQRegClass.contains(SrcReg)) {
2786     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2787                                        AArch64::qsub2, AArch64::qsub3};
2788     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2789                      Indices);
2790     return;
2791   }
2792 
2793   // Copy a QQQ register triple by copying the individual sub-registers.
2794   if (AArch64::QQQRegClass.contains(DestReg) &&
2795       AArch64::QQQRegClass.contains(SrcReg)) {
2796     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2797                                        AArch64::qsub2};
2798     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2799                      Indices);
2800     return;
2801   }
2802 
2803   // Copy a QQ register pair by copying the individual sub-registers.
2804   if (AArch64::QQRegClass.contains(DestReg) &&
2805       AArch64::QQRegClass.contains(SrcReg)) {
2806     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
2807     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2808                      Indices);
2809     return;
2810   }
2811 
2812   if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
2813       AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
2814     static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
2815     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
2816                     AArch64::XZR, Indices);
2817     return;
2818   }
2819 
2820   if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
2821       AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
2822     static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
2823     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
2824                     AArch64::WZR, Indices);
2825     return;
2826   }
2827 
2828   if (AArch64::FPR128RegClass.contains(DestReg) &&
2829       AArch64::FPR128RegClass.contains(SrcReg)) {
2830     if (Subtarget.hasNEON()) {
2831       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2832           .addReg(SrcReg)
2833           .addReg(SrcReg, getKillRegState(KillSrc));
2834     } else {
2835       BuildMI(MBB, I, DL, get(AArch64::STRQpre))
2836           .addReg(AArch64::SP, RegState::Define)
2837           .addReg(SrcReg, getKillRegState(KillSrc))
2838           .addReg(AArch64::SP)
2839           .addImm(-16);
2840       BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
2841           .addReg(AArch64::SP, RegState::Define)
2842           .addReg(DestReg, RegState::Define)
2843           .addReg(AArch64::SP)
2844           .addImm(16);
2845     }
2846     return;
2847   }
2848 
2849   if (AArch64::FPR64RegClass.contains(DestReg) &&
2850       AArch64::FPR64RegClass.contains(SrcReg)) {
2851     if (Subtarget.hasNEON()) {
2852       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
2853                                        &AArch64::FPR128RegClass);
2854       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
2855                                       &AArch64::FPR128RegClass);
2856       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2857           .addReg(SrcReg)
2858           .addReg(SrcReg, getKillRegState(KillSrc));
2859     } else {
2860       BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
2861           .addReg(SrcReg, getKillRegState(KillSrc));
2862     }
2863     return;
2864   }
2865 
2866   if (AArch64::FPR32RegClass.contains(DestReg) &&
2867       AArch64::FPR32RegClass.contains(SrcReg)) {
2868     if (Subtarget.hasNEON()) {
2869       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
2870                                        &AArch64::FPR128RegClass);
2871       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
2872                                       &AArch64::FPR128RegClass);
2873       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2874           .addReg(SrcReg)
2875           .addReg(SrcReg, getKillRegState(KillSrc));
2876     } else {
2877       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2878           .addReg(SrcReg, getKillRegState(KillSrc));
2879     }
2880     return;
2881   }
2882 
2883   if (AArch64::FPR16RegClass.contains(DestReg) &&
2884       AArch64::FPR16RegClass.contains(SrcReg)) {
2885     if (Subtarget.hasNEON()) {
2886       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2887                                        &AArch64::FPR128RegClass);
2888       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2889                                       &AArch64::FPR128RegClass);
2890       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2891           .addReg(SrcReg)
2892           .addReg(SrcReg, getKillRegState(KillSrc));
2893     } else {
2894       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2895                                        &AArch64::FPR32RegClass);
2896       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2897                                       &AArch64::FPR32RegClass);
2898       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2899           .addReg(SrcReg, getKillRegState(KillSrc));
2900     }
2901     return;
2902   }
2903 
2904   if (AArch64::FPR8RegClass.contains(DestReg) &&
2905       AArch64::FPR8RegClass.contains(SrcReg)) {
2906     if (Subtarget.hasNEON()) {
2907       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2908                                        &AArch64::FPR128RegClass);
2909       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2910                                       &AArch64::FPR128RegClass);
2911       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2912           .addReg(SrcReg)
2913           .addReg(SrcReg, getKillRegState(KillSrc));
2914     } else {
2915       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2916                                        &AArch64::FPR32RegClass);
2917       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2918                                       &AArch64::FPR32RegClass);
2919       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2920           .addReg(SrcReg, getKillRegState(KillSrc));
2921     }
2922     return;
2923   }
2924 
2925   // Copies between GPR64 and FPR64.
2926   if (AArch64::FPR64RegClass.contains(DestReg) &&
2927       AArch64::GPR64RegClass.contains(SrcReg)) {
2928     BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
2929         .addReg(SrcReg, getKillRegState(KillSrc));
2930     return;
2931   }
2932   if (AArch64::GPR64RegClass.contains(DestReg) &&
2933       AArch64::FPR64RegClass.contains(SrcReg)) {
2934     BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
2935         .addReg(SrcReg, getKillRegState(KillSrc));
2936     return;
2937   }
2938   // Copies between GPR32 and FPR32.
2939   if (AArch64::FPR32RegClass.contains(DestReg) &&
2940       AArch64::GPR32RegClass.contains(SrcReg)) {
2941     BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
2942         .addReg(SrcReg, getKillRegState(KillSrc));
2943     return;
2944   }
2945   if (AArch64::GPR32RegClass.contains(DestReg) &&
2946       AArch64::FPR32RegClass.contains(SrcReg)) {
2947     BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
2948         .addReg(SrcReg, getKillRegState(KillSrc));
2949     return;
2950   }
2951 
2952   if (DestReg == AArch64::NZCV) {
2953     assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
2954     BuildMI(MBB, I, DL, get(AArch64::MSR))
2955         .addImm(AArch64SysReg::NZCV)
2956         .addReg(SrcReg, getKillRegState(KillSrc))
2957         .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
2958     return;
2959   }
2960 
2961   if (SrcReg == AArch64::NZCV) {
2962     assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
2963     BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
2964         .addImm(AArch64SysReg::NZCV)
2965         .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
2966     return;
2967   }
2968 
2969   llvm_unreachable("unimplemented reg-to-reg copy");
2970 }
2971 
2972 static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
2973                                     MachineBasicBlock &MBB,
2974                                     MachineBasicBlock::iterator InsertBefore,
2975                                     const MCInstrDesc &MCID,
2976                                     Register SrcReg, bool IsKill,
2977                                     unsigned SubIdx0, unsigned SubIdx1, int FI,
2978                                     MachineMemOperand *MMO) {
2979   Register SrcReg0 = SrcReg;
2980   Register SrcReg1 = SrcReg;
2981   if (Register::isPhysicalRegister(SrcReg)) {
2982     SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
2983     SubIdx0 = 0;
2984     SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
2985     SubIdx1 = 0;
2986   }
2987   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
2988       .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
2989       .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
2990       .addFrameIndex(FI)
2991       .addImm(0)
2992       .addMemOperand(MMO);
2993 }
2994 
2995 void AArch64InstrInfo::storeRegToStackSlot(
2996     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
2997     bool isKill, int FI, const TargetRegisterClass *RC,
2998     const TargetRegisterInfo *TRI) const {
2999   MachineFunction &MF = *MBB.getParent();
3000   MachineFrameInfo &MFI = MF.getFrameInfo();
3001 
3002   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3003   MachineMemOperand *MMO =
3004       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
3005                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3006   unsigned Opc = 0;
3007   bool Offset = true;
3008   unsigned StackID = TargetStackID::Default;
3009   switch (TRI->getSpillSize(*RC)) {
3010   case 1:
3011     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3012       Opc = AArch64::STRBui;
3013     break;
3014   case 2:
3015     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3016       Opc = AArch64::STRHui;
3017     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3018       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3019       Opc = AArch64::STR_PXI;
3020       StackID = TargetStackID::SVEVector;
3021     }
3022     break;
3023   case 4:
3024     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3025       Opc = AArch64::STRWui;
3026       if (Register::isVirtualRegister(SrcReg))
3027         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
3028       else
3029         assert(SrcReg != AArch64::WSP);
3030     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3031       Opc = AArch64::STRSui;
3032     break;
3033   case 8:
3034     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3035       Opc = AArch64::STRXui;
3036       if (Register::isVirtualRegister(SrcReg))
3037         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3038       else
3039         assert(SrcReg != AArch64::SP);
3040     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3041       Opc = AArch64::STRDui;
3042     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3043       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3044                               get(AArch64::STPWi), SrcReg, isKill,
3045                               AArch64::sube32, AArch64::subo32, FI, MMO);
3046       return;
3047     }
3048     break;
3049   case 16:
3050     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3051       Opc = AArch64::STRQui;
3052     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3053       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3054       Opc = AArch64::ST1Twov1d;
3055       Offset = false;
3056     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3057       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
3058                               get(AArch64::STPXi), SrcReg, isKill,
3059                               AArch64::sube64, AArch64::subo64, FI, MMO);
3060       return;
3061     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3062       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3063       Opc = AArch64::STR_ZXI;
3064       StackID = TargetStackID::SVEVector;
3065     }
3066     break;
3067   case 24:
3068     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3069       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3070       Opc = AArch64::ST1Threev1d;
3071       Offset = false;
3072     }
3073     break;
3074   case 32:
3075     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3076       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3077       Opc = AArch64::ST1Fourv1d;
3078       Offset = false;
3079     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3080       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3081       Opc = AArch64::ST1Twov2d;
3082       Offset = false;
3083     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3084       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3085       Opc = AArch64::STR_ZZXI;
3086       StackID = TargetStackID::SVEVector;
3087     }
3088     break;
3089   case 48:
3090     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3091       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3092       Opc = AArch64::ST1Threev2d;
3093       Offset = false;
3094     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3095       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3096       Opc = AArch64::STR_ZZZXI;
3097       StackID = TargetStackID::SVEVector;
3098     }
3099     break;
3100   case 64:
3101     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3102       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3103       Opc = AArch64::ST1Fourv2d;
3104       Offset = false;
3105     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3106       assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3107       Opc = AArch64::STR_ZZZZXI;
3108       StackID = TargetStackID::SVEVector;
3109     }
3110     break;
3111   }
3112   assert(Opc && "Unknown register class");
3113   MFI.setStackID(FI, StackID);
3114 
3115   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3116                                      .addReg(SrcReg, getKillRegState(isKill))
3117                                      .addFrameIndex(FI);
3118 
3119   if (Offset)
3120     MI.addImm(0);
3121   MI.addMemOperand(MMO);
3122 }
3123 
3124 static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
3125                                      MachineBasicBlock &MBB,
3126                                      MachineBasicBlock::iterator InsertBefore,
3127                                      const MCInstrDesc &MCID,
3128                                      Register DestReg, unsigned SubIdx0,
3129                                      unsigned SubIdx1, int FI,
3130                                      MachineMemOperand *MMO) {
3131   Register DestReg0 = DestReg;
3132   Register DestReg1 = DestReg;
3133   bool IsUndef = true;
3134   if (Register::isPhysicalRegister(DestReg)) {
3135     DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
3136     SubIdx0 = 0;
3137     DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
3138     SubIdx1 = 0;
3139     IsUndef = false;
3140   }
3141   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3142       .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
3143       .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
3144       .addFrameIndex(FI)
3145       .addImm(0)
3146       .addMemOperand(MMO);
3147 }
3148 
3149 void AArch64InstrInfo::loadRegFromStackSlot(
3150     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
3151     int FI, const TargetRegisterClass *RC,
3152     const TargetRegisterInfo *TRI) const {
3153   MachineFunction &MF = *MBB.getParent();
3154   MachineFrameInfo &MFI = MF.getFrameInfo();
3155   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3156   MachineMemOperand *MMO =
3157       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
3158                               MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3159 
3160   unsigned Opc = 0;
3161   bool Offset = true;
3162   unsigned StackID = TargetStackID::Default;
3163   switch (TRI->getSpillSize(*RC)) {
3164   case 1:
3165     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3166       Opc = AArch64::LDRBui;
3167     break;
3168   case 2:
3169     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3170       Opc = AArch64::LDRHui;
3171     else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3172       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3173       Opc = AArch64::LDR_PXI;
3174       StackID = TargetStackID::SVEVector;
3175     }
3176     break;
3177   case 4:
3178     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3179       Opc = AArch64::LDRWui;
3180       if (Register::isVirtualRegister(DestReg))
3181         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
3182       else
3183         assert(DestReg != AArch64::WSP);
3184     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3185       Opc = AArch64::LDRSui;
3186     break;
3187   case 8:
3188     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3189       Opc = AArch64::LDRXui;
3190       if (Register::isVirtualRegister(DestReg))
3191         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
3192       else
3193         assert(DestReg != AArch64::SP);
3194     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3195       Opc = AArch64::LDRDui;
3196     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3197       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3198                                get(AArch64::LDPWi), DestReg, AArch64::sube32,
3199                                AArch64::subo32, FI, MMO);
3200       return;
3201     }
3202     break;
3203   case 16:
3204     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3205       Opc = AArch64::LDRQui;
3206     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3207       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3208       Opc = AArch64::LD1Twov1d;
3209       Offset = false;
3210     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3211       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3212                                get(AArch64::LDPXi), DestReg, AArch64::sube64,
3213                                AArch64::subo64, FI, MMO);
3214       return;
3215     } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3216       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3217       Opc = AArch64::LDR_ZXI;
3218       StackID = TargetStackID::SVEVector;
3219     }
3220     break;
3221   case 24:
3222     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3223       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3224       Opc = AArch64::LD1Threev1d;
3225       Offset = false;
3226     }
3227     break;
3228   case 32:
3229     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3230       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3231       Opc = AArch64::LD1Fourv1d;
3232       Offset = false;
3233     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3234       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3235       Opc = AArch64::LD1Twov2d;
3236       Offset = false;
3237     } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3238       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3239       Opc = AArch64::LDR_ZZXI;
3240       StackID = TargetStackID::SVEVector;
3241     }
3242     break;
3243   case 48:
3244     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3245       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3246       Opc = AArch64::LD1Threev2d;
3247       Offset = false;
3248     } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3249       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3250       Opc = AArch64::LDR_ZZZXI;
3251       StackID = TargetStackID::SVEVector;
3252     }
3253     break;
3254   case 64:
3255     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3256       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3257       Opc = AArch64::LD1Fourv2d;
3258       Offset = false;
3259     } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3260       assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3261       Opc = AArch64::LDR_ZZZZXI;
3262       StackID = TargetStackID::SVEVector;
3263     }
3264     break;
3265   }
3266 
3267   assert(Opc && "Unknown register class");
3268   MFI.setStackID(FI, StackID);
3269 
3270   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3271                                      .addReg(DestReg, getDefRegState(true))
3272                                      .addFrameIndex(FI);
3273   if (Offset)
3274     MI.addImm(0);
3275   MI.addMemOperand(MMO);
3276 }
3277 
3278 bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
3279                                            const MachineInstr &UseMI,
3280                                            const TargetRegisterInfo *TRI) {
3281   return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
3282                                          UseMI.getIterator()),
3283                 [TRI](const MachineInstr &I) {
3284                   return I.modifiesRegister(AArch64::NZCV, TRI) ||
3285                          I.readsRegister(AArch64::NZCV, TRI);
3286                 });
3287 }
3288 
3289 // Helper function to emit a frame offset adjustment from a given
3290 // pointer (SrcReg), stored into DestReg. This function is explicit
3291 // in that it requires the opcode.
3292 static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
3293                                MachineBasicBlock::iterator MBBI,
3294                                const DebugLoc &DL, unsigned DestReg,
3295                                unsigned SrcReg, int64_t Offset, unsigned Opc,
3296                                const TargetInstrInfo *TII,
3297                                MachineInstr::MIFlag Flag, bool NeedsWinCFI,
3298                                bool *HasWinCFI) {
3299   int Sign = 1;
3300   unsigned MaxEncoding, ShiftSize;
3301   switch (Opc) {
3302   case AArch64::ADDXri:
3303   case AArch64::ADDSXri:
3304   case AArch64::SUBXri:
3305   case AArch64::SUBSXri:
3306     MaxEncoding = 0xfff;
3307     ShiftSize = 12;
3308     break;
3309   case AArch64::ADDVL_XXI:
3310   case AArch64::ADDPL_XXI:
3311     MaxEncoding = 31;
3312     ShiftSize = 0;
3313     if (Offset < 0) {
3314       MaxEncoding = 32;
3315       Sign = -1;
3316       Offset = -Offset;
3317     }
3318     break;
3319   default:
3320     llvm_unreachable("Unsupported opcode");
3321   }
3322 
3323   // FIXME: If the offset won't fit in 24-bits, compute the offset into a
3324   // scratch register.  If DestReg is a virtual register, use it as the
3325   // scratch register; otherwise, create a new virtual register (to be
3326   // replaced by the scavenger at the end of PEI).  That case can be optimized
3327   // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
3328   // register can be loaded with offset%8 and the add/sub can use an extending
3329   // instruction with LSL#3.
3330   // Currently the function handles any offsets but generates a poor sequence
3331   // of code.
3332   //  assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
3333 
3334   const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
3335   Register TmpReg = DestReg;
3336   if (TmpReg == AArch64::XZR)
3337     TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
3338         &AArch64::GPR64RegClass);
3339   do {
3340     uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
3341     unsigned LocalShiftSize = 0;
3342     if (ThisVal > MaxEncoding) {
3343       ThisVal = ThisVal >> ShiftSize;
3344       LocalShiftSize = ShiftSize;
3345     }
3346     assert((ThisVal >> ShiftSize) <= MaxEncoding &&
3347            "Encoding cannot handle value that big");
3348 
3349     Offset -= ThisVal << LocalShiftSize;
3350     if (Offset == 0)
3351       TmpReg = DestReg;
3352     auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg)
3353                    .addReg(SrcReg)
3354                    .addImm(Sign * (int)ThisVal);
3355     if (ShiftSize)
3356       MBI = MBI.addImm(
3357           AArch64_AM::getShifterImm(AArch64_AM::LSL, LocalShiftSize));
3358     MBI = MBI.setMIFlag(Flag);
3359 
3360     if (NeedsWinCFI) {
3361       assert(Sign == 1 && "SEH directives should always have a positive sign");
3362       int Imm = (int)(ThisVal << LocalShiftSize);
3363       if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
3364           (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
3365         if (HasWinCFI)
3366           *HasWinCFI = true;
3367         if (Imm == 0)
3368           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
3369         else
3370           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
3371               .addImm(Imm)
3372               .setMIFlag(Flag);
3373         assert(Offset == 0 && "Expected remaining offset to be zero to "
3374                               "emit a single SEH directive");
3375       } else if (DestReg == AArch64::SP) {
3376         if (HasWinCFI)
3377           *HasWinCFI = true;
3378         assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
3379         BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
3380             .addImm(Imm)
3381             .setMIFlag(Flag);
3382       }
3383       if (HasWinCFI)
3384         *HasWinCFI = true;
3385     }
3386 
3387     SrcReg = TmpReg;
3388   } while (Offset);
3389 }
3390 
3391 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
3392                            MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
3393                            unsigned DestReg, unsigned SrcReg,
3394                            StackOffset Offset, const TargetInstrInfo *TII,
3395                            MachineInstr::MIFlag Flag, bool SetNZCV,
3396                            bool NeedsWinCFI, bool *HasWinCFI) {
3397   int64_t Bytes, NumPredicateVectors, NumDataVectors;
3398   Offset.getForFrameOffset(Bytes, NumPredicateVectors, NumDataVectors);
3399 
3400   // First emit non-scalable frame offsets, or a simple 'mov'.
3401   if (Bytes || (!Offset && SrcReg != DestReg)) {
3402     assert((DestReg != AArch64::SP || Bytes % 16 == 0) &&
3403            "SP increment/decrement not 16-byte aligned");
3404     unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
3405     if (Bytes < 0) {
3406       Bytes = -Bytes;
3407       Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
3408     }
3409     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
3410                        NeedsWinCFI, HasWinCFI);
3411     SrcReg = DestReg;
3412   }
3413 
3414   assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
3415          "SetNZCV not supported with SVE vectors");
3416   assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
3417          "WinCFI not supported with SVE vectors");
3418 
3419   if (NumDataVectors) {
3420     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
3421                        AArch64::ADDVL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3422     SrcReg = DestReg;
3423   }
3424 
3425   if (NumPredicateVectors) {
3426     assert(DestReg != AArch64::SP && "Unaligned access to SP");
3427     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
3428                        AArch64::ADDPL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3429   }
3430 }
3431 
3432 MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
3433     MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
3434     MachineBasicBlock::iterator InsertPt, int FrameIndex,
3435     LiveIntervals *LIS, VirtRegMap *VRM) const {
3436   // This is a bit of a hack. Consider this instruction:
3437   //
3438   //   %0 = COPY %sp; GPR64all:%0
3439   //
3440   // We explicitly chose GPR64all for the virtual register so such a copy might
3441   // be eliminated by RegisterCoalescer. However, that may not be possible, and
3442   // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
3443   // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
3444   //
3445   // To prevent that, we are going to constrain the %0 register class here.
3446   //
3447   // <rdar://problem/11522048>
3448   //
3449   if (MI.isFullCopy()) {
3450     Register DstReg = MI.getOperand(0).getReg();
3451     Register SrcReg = MI.getOperand(1).getReg();
3452     if (SrcReg == AArch64::SP && Register::isVirtualRegister(DstReg)) {
3453       MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
3454       return nullptr;
3455     }
3456     if (DstReg == AArch64::SP && Register::isVirtualRegister(SrcReg)) {
3457       MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3458       return nullptr;
3459     }
3460   }
3461 
3462   // Handle the case where a copy is being spilled or filled but the source
3463   // and destination register class don't match.  For example:
3464   //
3465   //   %0 = COPY %xzr; GPR64common:%0
3466   //
3467   // In this case we can still safely fold away the COPY and generate the
3468   // following spill code:
3469   //
3470   //   STRXui %xzr, %stack.0
3471   //
3472   // This also eliminates spilled cross register class COPYs (e.g. between x and
3473   // d regs) of the same size.  For example:
3474   //
3475   //   %0 = COPY %1; GPR64:%0, FPR64:%1
3476   //
3477   // will be filled as
3478   //
3479   //   LDRDui %0, fi<#0>
3480   //
3481   // instead of
3482   //
3483   //   LDRXui %Temp, fi<#0>
3484   //   %0 = FMOV %Temp
3485   //
3486   if (MI.isCopy() && Ops.size() == 1 &&
3487       // Make sure we're only folding the explicit COPY defs/uses.
3488       (Ops[0] == 0 || Ops[0] == 1)) {
3489     bool IsSpill = Ops[0] == 0;
3490     bool IsFill = !IsSpill;
3491     const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
3492     const MachineRegisterInfo &MRI = MF.getRegInfo();
3493     MachineBasicBlock &MBB = *MI.getParent();
3494     const MachineOperand &DstMO = MI.getOperand(0);
3495     const MachineOperand &SrcMO = MI.getOperand(1);
3496     Register DstReg = DstMO.getReg();
3497     Register SrcReg = SrcMO.getReg();
3498     // This is slightly expensive to compute for physical regs since
3499     // getMinimalPhysRegClass is slow.
3500     auto getRegClass = [&](unsigned Reg) {
3501       return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
3502                                               : TRI.getMinimalPhysRegClass(Reg);
3503     };
3504 
3505     if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
3506       assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
3507                  TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
3508              "Mismatched register size in non subreg COPY");
3509       if (IsSpill)
3510         storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
3511                             getRegClass(SrcReg), &TRI);
3512       else
3513         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
3514                              getRegClass(DstReg), &TRI);
3515       return &*--InsertPt;
3516     }
3517 
3518     // Handle cases like spilling def of:
3519     //
3520     //   %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
3521     //
3522     // where the physical register source can be widened and stored to the full
3523     // virtual reg destination stack slot, in this case producing:
3524     //
3525     //   STRXui %xzr, %stack.0
3526     //
3527     if (IsSpill && DstMO.isUndef() && Register::isPhysicalRegister(SrcReg)) {
3528       assert(SrcMO.getSubReg() == 0 &&
3529              "Unexpected subreg on physical register");
3530       const TargetRegisterClass *SpillRC;
3531       unsigned SpillSubreg;
3532       switch (DstMO.getSubReg()) {
3533       default:
3534         SpillRC = nullptr;
3535         break;
3536       case AArch64::sub_32:
3537       case AArch64::ssub:
3538         if (AArch64::GPR32RegClass.contains(SrcReg)) {
3539           SpillRC = &AArch64::GPR64RegClass;
3540           SpillSubreg = AArch64::sub_32;
3541         } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
3542           SpillRC = &AArch64::FPR64RegClass;
3543           SpillSubreg = AArch64::ssub;
3544         } else
3545           SpillRC = nullptr;
3546         break;
3547       case AArch64::dsub:
3548         if (AArch64::FPR64RegClass.contains(SrcReg)) {
3549           SpillRC = &AArch64::FPR128RegClass;
3550           SpillSubreg = AArch64::dsub;
3551         } else
3552           SpillRC = nullptr;
3553         break;
3554       }
3555 
3556       if (SpillRC)
3557         if (unsigned WidenedSrcReg =
3558                 TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
3559           storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
3560                               FrameIndex, SpillRC, &TRI);
3561           return &*--InsertPt;
3562         }
3563     }
3564 
3565     // Handle cases like filling use of:
3566     //
3567     //   %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
3568     //
3569     // where we can load the full virtual reg source stack slot, into the subreg
3570     // destination, in this case producing:
3571     //
3572     //   LDRWui %0:sub_32<def,read-undef>, %stack.0
3573     //
3574     if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
3575       const TargetRegisterClass *FillRC;
3576       switch (DstMO.getSubReg()) {
3577       default:
3578         FillRC = nullptr;
3579         break;
3580       case AArch64::sub_32:
3581         FillRC = &AArch64::GPR32RegClass;
3582         break;
3583       case AArch64::ssub:
3584         FillRC = &AArch64::FPR32RegClass;
3585         break;
3586       case AArch64::dsub:
3587         FillRC = &AArch64::FPR64RegClass;
3588         break;
3589       }
3590 
3591       if (FillRC) {
3592         assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
3593                    TRI.getRegSizeInBits(*FillRC) &&
3594                "Mismatched regclass size on folded subreg COPY");
3595         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC, &TRI);
3596         MachineInstr &LoadMI = *--InsertPt;
3597         MachineOperand &LoadDst = LoadMI.getOperand(0);
3598         assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
3599         LoadDst.setSubReg(DstMO.getSubReg());
3600         LoadDst.setIsUndef();
3601         return &LoadMI;
3602       }
3603     }
3604   }
3605 
3606   // Cannot fold.
3607   return nullptr;
3608 }
3609 
3610 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
3611                                     StackOffset &SOffset,
3612                                     bool *OutUseUnscaledOp,
3613                                     unsigned *OutUnscaledOp,
3614                                     int64_t *EmittableOffset) {
3615   // Set output values in case of early exit.
3616   if (EmittableOffset)
3617     *EmittableOffset = 0;
3618   if (OutUseUnscaledOp)
3619     *OutUseUnscaledOp = false;
3620   if (OutUnscaledOp)
3621     *OutUnscaledOp = 0;
3622 
3623   // Exit early for structured vector spills/fills as they can't take an
3624   // immediate offset.
3625   switch (MI.getOpcode()) {
3626   default:
3627     break;
3628   case AArch64::LD1Twov2d:
3629   case AArch64::LD1Threev2d:
3630   case AArch64::LD1Fourv2d:
3631   case AArch64::LD1Twov1d:
3632   case AArch64::LD1Threev1d:
3633   case AArch64::LD1Fourv1d:
3634   case AArch64::ST1Twov2d:
3635   case AArch64::ST1Threev2d:
3636   case AArch64::ST1Fourv2d:
3637   case AArch64::ST1Twov1d:
3638   case AArch64::ST1Threev1d:
3639   case AArch64::ST1Fourv1d:
3640   case AArch64::IRG:
3641   case AArch64::IRGstack:
3642   case AArch64::STGloop:
3643   case AArch64::STZGloop:
3644     return AArch64FrameOffsetCannotUpdate;
3645   }
3646 
3647   // Get the min/max offset and the scale.
3648   TypeSize ScaleValue(0U, false);
3649   unsigned Width;
3650   int64_t MinOff, MaxOff;
3651   if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
3652                                       MaxOff))
3653     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3654 
3655   // Construct the complete offset.
3656   bool IsMulVL = ScaleValue.isScalable();
3657   unsigned Scale = ScaleValue.getKnownMinSize();
3658   int64_t Offset = IsMulVL ? SOffset.getScalableBytes() : SOffset.getBytes();
3659 
3660   const MachineOperand &ImmOpnd =
3661       MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
3662   Offset += ImmOpnd.getImm() * Scale;
3663 
3664   // If the offset doesn't match the scale, we rewrite the instruction to
3665   // use the unscaled instruction instead. Likewise, if we have a negative
3666   // offset and there is an unscaled op to use.
3667   Optional<unsigned> UnscaledOp =
3668       AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
3669   bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
3670   if (useUnscaledOp &&
3671       !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
3672                                       MaxOff))
3673     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3674 
3675   Scale = ScaleValue.getKnownMinSize();
3676   assert(IsMulVL == ScaleValue.isScalable() &&
3677          "Unscaled opcode has different value for scalable");
3678 
3679   int64_t Remainder = Offset % Scale;
3680   assert(!(Remainder && useUnscaledOp) &&
3681          "Cannot have remainder when using unscaled op");
3682 
3683   assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
3684   int64_t NewOffset = Offset / Scale;
3685   if (MinOff <= NewOffset && NewOffset <= MaxOff)
3686     Offset = Remainder;
3687   else {
3688     NewOffset = NewOffset < 0 ? MinOff : MaxOff;
3689     Offset = Offset - NewOffset * Scale + Remainder;
3690   }
3691 
3692   if (EmittableOffset)
3693     *EmittableOffset = NewOffset;
3694   if (OutUseUnscaledOp)
3695     *OutUseUnscaledOp = useUnscaledOp;
3696   if (OutUnscaledOp && UnscaledOp)
3697     *OutUnscaledOp = *UnscaledOp;
3698 
3699   if (IsMulVL)
3700     SOffset = StackOffset(Offset, MVT::nxv1i8) +
3701               StackOffset(SOffset.getBytes(), MVT::i8);
3702   else
3703     SOffset = StackOffset(Offset, MVT::i8) +
3704               StackOffset(SOffset.getScalableBytes(), MVT::nxv1i8);
3705   return AArch64FrameOffsetCanUpdate |
3706          (SOffset ? 0 : AArch64FrameOffsetIsLegal);
3707 }
3708 
3709 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
3710                                     unsigned FrameReg, StackOffset &Offset,
3711                                     const AArch64InstrInfo *TII) {
3712   unsigned Opcode = MI.getOpcode();
3713   unsigned ImmIdx = FrameRegIdx + 1;
3714 
3715   if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
3716     Offset += StackOffset(MI.getOperand(ImmIdx).getImm(), MVT::i8);
3717     emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
3718                     MI.getOperand(0).getReg(), FrameReg, Offset, TII,
3719                     MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
3720     MI.eraseFromParent();
3721     Offset = StackOffset();
3722     return true;
3723   }
3724 
3725   int64_t NewOffset;
3726   unsigned UnscaledOp;
3727   bool UseUnscaledOp;
3728   int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
3729                                          &UnscaledOp, &NewOffset);
3730   if (Status & AArch64FrameOffsetCanUpdate) {
3731     if (Status & AArch64FrameOffsetIsLegal)
3732       // Replace the FrameIndex with FrameReg.
3733       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
3734     if (UseUnscaledOp)
3735       MI.setDesc(TII->get(UnscaledOp));
3736 
3737     MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
3738     return !Offset;
3739   }
3740 
3741   return false;
3742 }
3743 
3744 void AArch64InstrInfo::getNoop(MCInst &NopInst) const {
3745   NopInst.setOpcode(AArch64::HINT);
3746   NopInst.addOperand(MCOperand::createImm(0));
3747 }
3748 
3749 // AArch64 supports MachineCombiner.
3750 bool AArch64InstrInfo::useMachineCombiner() const { return true; }
3751 
3752 // True when Opc sets flag
3753 static bool isCombineInstrSettingFlag(unsigned Opc) {
3754   switch (Opc) {
3755   case AArch64::ADDSWrr:
3756   case AArch64::ADDSWri:
3757   case AArch64::ADDSXrr:
3758   case AArch64::ADDSXri:
3759   case AArch64::SUBSWrr:
3760   case AArch64::SUBSXrr:
3761   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3762   case AArch64::SUBSWri:
3763   case AArch64::SUBSXri:
3764     return true;
3765   default:
3766     break;
3767   }
3768   return false;
3769 }
3770 
3771 // 32b Opcodes that can be combined with a MUL
3772 static bool isCombineInstrCandidate32(unsigned Opc) {
3773   switch (Opc) {
3774   case AArch64::ADDWrr:
3775   case AArch64::ADDWri:
3776   case AArch64::SUBWrr:
3777   case AArch64::ADDSWrr:
3778   case AArch64::ADDSWri:
3779   case AArch64::SUBSWrr:
3780   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3781   case AArch64::SUBWri:
3782   case AArch64::SUBSWri:
3783     return true;
3784   default:
3785     break;
3786   }
3787   return false;
3788 }
3789 
3790 // 64b Opcodes that can be combined with a MUL
3791 static bool isCombineInstrCandidate64(unsigned Opc) {
3792   switch (Opc) {
3793   case AArch64::ADDXrr:
3794   case AArch64::ADDXri:
3795   case AArch64::SUBXrr:
3796   case AArch64::ADDSXrr:
3797   case AArch64::ADDSXri:
3798   case AArch64::SUBSXrr:
3799   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3800   case AArch64::SUBXri:
3801   case AArch64::SUBSXri:
3802   case AArch64::ADDv8i8:
3803   case AArch64::ADDv16i8:
3804   case AArch64::ADDv4i16:
3805   case AArch64::ADDv8i16:
3806   case AArch64::ADDv2i32:
3807   case AArch64::ADDv4i32:
3808   case AArch64::SUBv8i8:
3809   case AArch64::SUBv16i8:
3810   case AArch64::SUBv4i16:
3811   case AArch64::SUBv8i16:
3812   case AArch64::SUBv2i32:
3813   case AArch64::SUBv4i32:
3814     return true;
3815   default:
3816     break;
3817   }
3818   return false;
3819 }
3820 
3821 // FP Opcodes that can be combined with a FMUL
3822 static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
3823   switch (Inst.getOpcode()) {
3824   default:
3825     break;
3826   case AArch64::FADDHrr:
3827   case AArch64::FADDSrr:
3828   case AArch64::FADDDrr:
3829   case AArch64::FADDv4f16:
3830   case AArch64::FADDv8f16:
3831   case AArch64::FADDv2f32:
3832   case AArch64::FADDv2f64:
3833   case AArch64::FADDv4f32:
3834   case AArch64::FSUBHrr:
3835   case AArch64::FSUBSrr:
3836   case AArch64::FSUBDrr:
3837   case AArch64::FSUBv4f16:
3838   case AArch64::FSUBv8f16:
3839   case AArch64::FSUBv2f32:
3840   case AArch64::FSUBv2f64:
3841   case AArch64::FSUBv4f32:
3842     TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
3843     return (Options.UnsafeFPMath ||
3844             Options.AllowFPOpFusion == FPOpFusion::Fast);
3845   }
3846   return false;
3847 }
3848 
3849 // Opcodes that can be combined with a MUL
3850 static bool isCombineInstrCandidate(unsigned Opc) {
3851   return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
3852 }
3853 
3854 //
3855 // Utility routine that checks if \param MO is defined by an
3856 // \param CombineOpc instruction in the basic block \param MBB
3857 static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
3858                        unsigned CombineOpc, unsigned ZeroReg = 0,
3859                        bool CheckZeroReg = false) {
3860   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3861   MachineInstr *MI = nullptr;
3862 
3863   if (MO.isReg() && Register::isVirtualRegister(MO.getReg()))
3864     MI = MRI.getUniqueVRegDef(MO.getReg());
3865   // And it needs to be in the trace (otherwise, it won't have a depth).
3866   if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc)
3867     return false;
3868   // Must only used by the user we combine with.
3869   if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
3870     return false;
3871 
3872   if (CheckZeroReg) {
3873     assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
3874            MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
3875            MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
3876     // The third input reg must be zero.
3877     if (MI->getOperand(3).getReg() != ZeroReg)
3878       return false;
3879   }
3880 
3881   return true;
3882 }
3883 
3884 //
3885 // Is \param MO defined by an integer multiply and can be combined?
3886 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3887                               unsigned MulOpc, unsigned ZeroReg) {
3888   return canCombine(MBB, MO, MulOpc, ZeroReg, true);
3889 }
3890 
3891 //
3892 // Is \param MO defined by a floating-point multiply and can be combined?
3893 static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3894                                unsigned MulOpc) {
3895   return canCombine(MBB, MO, MulOpc);
3896 }
3897 
3898 // TODO: There are many more machine instruction opcodes to match:
3899 //       1. Other data types (integer, vectors)
3900 //       2. Other math / logic operations (xor, or)
3901 //       3. Other forms of the same operation (intrinsics and other variants)
3902 bool AArch64InstrInfo::isAssociativeAndCommutative(
3903     const MachineInstr &Inst) const {
3904   switch (Inst.getOpcode()) {
3905   case AArch64::FADDDrr:
3906   case AArch64::FADDSrr:
3907   case AArch64::FADDv2f32:
3908   case AArch64::FADDv2f64:
3909   case AArch64::FADDv4f32:
3910   case AArch64::FMULDrr:
3911   case AArch64::FMULSrr:
3912   case AArch64::FMULX32:
3913   case AArch64::FMULX64:
3914   case AArch64::FMULXv2f32:
3915   case AArch64::FMULXv2f64:
3916   case AArch64::FMULXv4f32:
3917   case AArch64::FMULv2f32:
3918   case AArch64::FMULv2f64:
3919   case AArch64::FMULv4f32:
3920     return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath;
3921   default:
3922     return false;
3923   }
3924 }
3925 
3926 /// Find instructions that can be turned into madd.
3927 static bool getMaddPatterns(MachineInstr &Root,
3928                             SmallVectorImpl<MachineCombinerPattern> &Patterns) {
3929   unsigned Opc = Root.getOpcode();
3930   MachineBasicBlock &MBB = *Root.getParent();
3931   bool Found = false;
3932 
3933   if (!isCombineInstrCandidate(Opc))
3934     return false;
3935   if (isCombineInstrSettingFlag(Opc)) {
3936     int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
3937     // When NZCV is live bail out.
3938     if (Cmp_NZCV == -1)
3939       return false;
3940     unsigned NewOpc = convertToNonFlagSettingOpc(Root);
3941     // When opcode can't change bail out.
3942     // CHECKME: do we miss any cases for opcode conversion?
3943     if (NewOpc == Opc)
3944       return false;
3945     Opc = NewOpc;
3946   }
3947 
3948   auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
3949                       MachineCombinerPattern Pattern) {
3950     if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
3951       Patterns.push_back(Pattern);
3952       Found = true;
3953     }
3954   };
3955 
3956   auto setVFound = [&](int Opcode, int Operand, MachineCombinerPattern Pattern) {
3957     if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
3958       Patterns.push_back(Pattern);
3959       Found = true;
3960     }
3961   };
3962 
3963   typedef MachineCombinerPattern MCP;
3964 
3965   switch (Opc) {
3966   default:
3967     break;
3968   case AArch64::ADDWrr:
3969     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
3970            "ADDWrr does not have register operands");
3971     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
3972     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
3973     break;
3974   case AArch64::ADDXrr:
3975     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
3976     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
3977     break;
3978   case AArch64::SUBWrr:
3979     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
3980     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
3981     break;
3982   case AArch64::SUBXrr:
3983     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
3984     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
3985     break;
3986   case AArch64::ADDWri:
3987     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
3988     break;
3989   case AArch64::ADDXri:
3990     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
3991     break;
3992   case AArch64::SUBWri:
3993     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
3994     break;
3995   case AArch64::SUBXri:
3996     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
3997     break;
3998   case AArch64::ADDv8i8:
3999     setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
4000     setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
4001     break;
4002   case AArch64::ADDv16i8:
4003     setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
4004     setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
4005     break;
4006   case AArch64::ADDv4i16:
4007     setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
4008     setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
4009     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
4010     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
4011     break;
4012   case AArch64::ADDv8i16:
4013     setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
4014     setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
4015     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
4016     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
4017     break;
4018   case AArch64::ADDv2i32:
4019     setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
4020     setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
4021     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
4022     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
4023     break;
4024   case AArch64::ADDv4i32:
4025     setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
4026     setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
4027     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
4028     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
4029     break;
4030   case AArch64::SUBv8i8:
4031     setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
4032     setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
4033     break;
4034   case AArch64::SUBv16i8:
4035     setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
4036     setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
4037     break;
4038   case AArch64::SUBv4i16:
4039     setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
4040     setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
4041     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
4042     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
4043     break;
4044   case AArch64::SUBv8i16:
4045     setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
4046     setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
4047     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
4048     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
4049     break;
4050   case AArch64::SUBv2i32:
4051     setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
4052     setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
4053     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
4054     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
4055     break;
4056   case AArch64::SUBv4i32:
4057     setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
4058     setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
4059     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
4060     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
4061     break;
4062   }
4063   return Found;
4064 }
4065 /// Floating-Point Support
4066 
4067 /// Find instructions that can be turned into madd.
4068 static bool getFMAPatterns(MachineInstr &Root,
4069                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {
4070 
4071   if (!isCombineInstrCandidateFP(Root))
4072     return false;
4073 
4074   MachineBasicBlock &MBB = *Root.getParent();
4075   bool Found = false;
4076 
4077   auto Match = [&](int Opcode, int Operand,
4078                    MachineCombinerPattern Pattern) -> bool {
4079     if (canCombineWithFMUL(MBB, Root.getOperand(Operand), Opcode)) {
4080       Patterns.push_back(Pattern);
4081       return true;
4082     }
4083     return false;
4084   };
4085 
4086   typedef MachineCombinerPattern MCP;
4087 
4088   switch (Root.getOpcode()) {
4089   default:
4090     assert(false && "Unsupported FP instruction in combiner\n");
4091     break;
4092   case AArch64::FADDHrr:
4093     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4094            "FADDHrr does not have register operands");
4095 
4096     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
4097     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
4098     break;
4099   case AArch64::FADDSrr:
4100     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
4101            "FADDSrr does not have register operands");
4102 
4103     Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
4104              Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
4105 
4106     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
4107              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
4108     break;
4109   case AArch64::FADDDrr:
4110     Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
4111              Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
4112 
4113     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
4114              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
4115     break;
4116   case AArch64::FADDv4f16:
4117     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
4118              Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
4119 
4120     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
4121              Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
4122     break;
4123   case AArch64::FADDv8f16:
4124     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
4125              Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
4126 
4127     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
4128              Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
4129     break;
4130   case AArch64::FADDv2f32:
4131     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
4132              Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
4133 
4134     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
4135              Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
4136     break;
4137   case AArch64::FADDv2f64:
4138     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
4139              Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
4140 
4141     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
4142              Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
4143     break;
4144   case AArch64::FADDv4f32:
4145     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
4146              Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
4147 
4148     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
4149              Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
4150     break;
4151   case AArch64::FSUBHrr:
4152     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
4153     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
4154     Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
4155     break;
4156   case AArch64::FSUBSrr:
4157     Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
4158 
4159     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
4160              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
4161 
4162     Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
4163     break;
4164   case AArch64::FSUBDrr:
4165     Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
4166 
4167     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
4168              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
4169 
4170     Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
4171     break;
4172   case AArch64::FSUBv4f16:
4173     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
4174              Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
4175 
4176     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
4177              Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
4178     break;
4179   case AArch64::FSUBv8f16:
4180     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
4181              Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
4182 
4183     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
4184              Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
4185     break;
4186   case AArch64::FSUBv2f32:
4187     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
4188              Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
4189 
4190     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
4191              Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
4192     break;
4193   case AArch64::FSUBv2f64:
4194     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
4195              Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
4196 
4197     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
4198              Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
4199     break;
4200   case AArch64::FSUBv4f32:
4201     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
4202              Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
4203 
4204     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
4205              Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
4206     break;
4207   }
4208   return Found;
4209 }
4210 
4211 /// Return true when a code sequence can improve throughput. It
4212 /// should be called only for instructions in loops.
4213 /// \param Pattern - combiner pattern
4214 bool AArch64InstrInfo::isThroughputPattern(
4215     MachineCombinerPattern Pattern) const {
4216   switch (Pattern) {
4217   default:
4218     break;
4219   case MachineCombinerPattern::FMULADDH_OP1:
4220   case MachineCombinerPattern::FMULADDH_OP2:
4221   case MachineCombinerPattern::FMULSUBH_OP1:
4222   case MachineCombinerPattern::FMULSUBH_OP2:
4223   case MachineCombinerPattern::FMULADDS_OP1:
4224   case MachineCombinerPattern::FMULADDS_OP2:
4225   case MachineCombinerPattern::FMULSUBS_OP1:
4226   case MachineCombinerPattern::FMULSUBS_OP2:
4227   case MachineCombinerPattern::FMULADDD_OP1:
4228   case MachineCombinerPattern::FMULADDD_OP2:
4229   case MachineCombinerPattern::FMULSUBD_OP1:
4230   case MachineCombinerPattern::FMULSUBD_OP2:
4231   case MachineCombinerPattern::FNMULSUBH_OP1:
4232   case MachineCombinerPattern::FNMULSUBS_OP1:
4233   case MachineCombinerPattern::FNMULSUBD_OP1:
4234   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
4235   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
4236   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
4237   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
4238   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4239   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4240   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
4241   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
4242   case MachineCombinerPattern::FMLAv4f16_OP2:
4243   case MachineCombinerPattern::FMLAv4f16_OP1:
4244   case MachineCombinerPattern::FMLAv8f16_OP1:
4245   case MachineCombinerPattern::FMLAv8f16_OP2:
4246   case MachineCombinerPattern::FMLAv2f32_OP2:
4247   case MachineCombinerPattern::FMLAv2f32_OP1:
4248   case MachineCombinerPattern::FMLAv2f64_OP1:
4249   case MachineCombinerPattern::FMLAv2f64_OP2:
4250   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
4251   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
4252   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
4253   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
4254   case MachineCombinerPattern::FMLAv4f32_OP1:
4255   case MachineCombinerPattern::FMLAv4f32_OP2:
4256   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
4257   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
4258   case MachineCombinerPattern::FMLSv4i16_indexed_OP1:
4259   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
4260   case MachineCombinerPattern::FMLSv8i16_indexed_OP1:
4261   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
4262   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
4263   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
4264   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
4265   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
4266   case MachineCombinerPattern::FMLSv4f16_OP1:
4267   case MachineCombinerPattern::FMLSv4f16_OP2:
4268   case MachineCombinerPattern::FMLSv8f16_OP1:
4269   case MachineCombinerPattern::FMLSv8f16_OP2:
4270   case MachineCombinerPattern::FMLSv2f32_OP2:
4271   case MachineCombinerPattern::FMLSv2f64_OP2:
4272   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
4273   case MachineCombinerPattern::FMLSv4f32_OP2:
4274   case MachineCombinerPattern::MULADDv8i8_OP1:
4275   case MachineCombinerPattern::MULADDv8i8_OP2:
4276   case MachineCombinerPattern::MULADDv16i8_OP1:
4277   case MachineCombinerPattern::MULADDv16i8_OP2:
4278   case MachineCombinerPattern::MULADDv4i16_OP1:
4279   case MachineCombinerPattern::MULADDv4i16_OP2:
4280   case MachineCombinerPattern::MULADDv8i16_OP1:
4281   case MachineCombinerPattern::MULADDv8i16_OP2:
4282   case MachineCombinerPattern::MULADDv2i32_OP1:
4283   case MachineCombinerPattern::MULADDv2i32_OP2:
4284   case MachineCombinerPattern::MULADDv4i32_OP1:
4285   case MachineCombinerPattern::MULADDv4i32_OP2:
4286   case MachineCombinerPattern::MULSUBv8i8_OP1:
4287   case MachineCombinerPattern::MULSUBv8i8_OP2:
4288   case MachineCombinerPattern::MULSUBv16i8_OP1:
4289   case MachineCombinerPattern::MULSUBv16i8_OP2:
4290   case MachineCombinerPattern::MULSUBv4i16_OP1:
4291   case MachineCombinerPattern::MULSUBv4i16_OP2:
4292   case MachineCombinerPattern::MULSUBv8i16_OP1:
4293   case MachineCombinerPattern::MULSUBv8i16_OP2:
4294   case MachineCombinerPattern::MULSUBv2i32_OP1:
4295   case MachineCombinerPattern::MULSUBv2i32_OP2:
4296   case MachineCombinerPattern::MULSUBv4i32_OP1:
4297   case MachineCombinerPattern::MULSUBv4i32_OP2:
4298   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4299   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4300   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4301   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4302   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4303   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4304   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4305   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4306   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4307   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4308   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4309   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4310   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4311   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4312   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4313   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4314     return true;
4315   } // end switch (Pattern)
4316   return false;
4317 }
4318 /// Return true when there is potentially a faster code sequence for an
4319 /// instruction chain ending in \p Root. All potential patterns are listed in
4320 /// the \p Pattern vector. Pattern should be sorted in priority order since the
4321 /// pattern evaluator stops checking as soon as it finds a faster sequence.
4322 
4323 bool AArch64InstrInfo::getMachineCombinerPatterns(
4324     MachineInstr &Root,
4325     SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
4326   // Integer patterns
4327   if (getMaddPatterns(Root, Patterns))
4328     return true;
4329   // Floating point patterns
4330   if (getFMAPatterns(Root, Patterns))
4331     return true;
4332 
4333   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
4334 }
4335 
4336 enum class FMAInstKind { Default, Indexed, Accumulator };
4337 /// genFusedMultiply - Generate fused multiply instructions.
4338 /// This function supports both integer and floating point instructions.
4339 /// A typical example:
4340 ///  F|MUL I=A,B,0
4341 ///  F|ADD R,I,C
4342 ///  ==> F|MADD R,A,B,C
4343 /// \param MF Containing MachineFunction
4344 /// \param MRI Register information
4345 /// \param TII Target information
4346 /// \param Root is the F|ADD instruction
4347 /// \param [out] InsInstrs is a vector of machine instructions and will
4348 /// contain the generated madd instruction
4349 /// \param IdxMulOpd is index of operand in Root that is the result of
4350 /// the F|MUL. In the example above IdxMulOpd is 1.
4351 /// \param MaddOpc the opcode fo the f|madd instruction
4352 /// \param RC Register class of operands
4353 /// \param kind of fma instruction (addressing mode) to be generated
4354 /// \param ReplacedAddend is the result register from the instruction
4355 /// replacing the non-combined operand, if any.
4356 static MachineInstr *
4357 genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
4358                  const TargetInstrInfo *TII, MachineInstr &Root,
4359                  SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
4360                  unsigned MaddOpc, const TargetRegisterClass *RC,
4361                  FMAInstKind kind = FMAInstKind::Default,
4362                  const Register *ReplacedAddend = nullptr) {
4363   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4364 
4365   unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
4366   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4367   Register ResultReg = Root.getOperand(0).getReg();
4368   Register SrcReg0 = MUL->getOperand(1).getReg();
4369   bool Src0IsKill = MUL->getOperand(1).isKill();
4370   Register SrcReg1 = MUL->getOperand(2).getReg();
4371   bool Src1IsKill = MUL->getOperand(2).isKill();
4372 
4373   unsigned SrcReg2;
4374   bool Src2IsKill;
4375   if (ReplacedAddend) {
4376     // If we just generated a new addend, we must be it's only use.
4377     SrcReg2 = *ReplacedAddend;
4378     Src2IsKill = true;
4379   } else {
4380     SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
4381     Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
4382   }
4383 
4384   if (Register::isVirtualRegister(ResultReg))
4385     MRI.constrainRegClass(ResultReg, RC);
4386   if (Register::isVirtualRegister(SrcReg0))
4387     MRI.constrainRegClass(SrcReg0, RC);
4388   if (Register::isVirtualRegister(SrcReg1))
4389     MRI.constrainRegClass(SrcReg1, RC);
4390   if (Register::isVirtualRegister(SrcReg2))
4391     MRI.constrainRegClass(SrcReg2, RC);
4392 
4393   MachineInstrBuilder MIB;
4394   if (kind == FMAInstKind::Default)
4395     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4396               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4397               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4398               .addReg(SrcReg2, getKillRegState(Src2IsKill));
4399   else if (kind == FMAInstKind::Indexed)
4400     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4401               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4402               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4403               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4404               .addImm(MUL->getOperand(3).getImm());
4405   else if (kind == FMAInstKind::Accumulator)
4406     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4407               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4408               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4409               .addReg(SrcReg1, getKillRegState(Src1IsKill));
4410   else
4411     assert(false && "Invalid FMA instruction kind \n");
4412   // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
4413   InsInstrs.push_back(MIB);
4414   return MUL;
4415 }
4416 
4417 /// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
4418 /// instructions.
4419 ///
4420 /// \see genFusedMultiply
4421 static MachineInstr *genFusedMultiplyAcc(
4422     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4423     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4424     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4425   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4426                           FMAInstKind::Accumulator);
4427 }
4428 
4429 /// genNeg - Helper to generate an intermediate negation of the second operand
4430 /// of Root
4431 static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
4432                        const TargetInstrInfo *TII, MachineInstr &Root,
4433                        SmallVectorImpl<MachineInstr *> &InsInstrs,
4434                        DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
4435                        unsigned MnegOpc, const TargetRegisterClass *RC) {
4436   Register NewVR = MRI.createVirtualRegister(RC);
4437   MachineInstrBuilder MIB =
4438       BuildMI(MF, Root.getDebugLoc(), TII->get(MnegOpc), NewVR)
4439           .add(Root.getOperand(2));
4440   InsInstrs.push_back(MIB);
4441 
4442   assert(InstrIdxForVirtReg.empty());
4443   InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4444 
4445   return NewVR;
4446 }
4447 
4448 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4449 /// instructions with an additional negation of the accumulator
4450 static MachineInstr *genFusedMultiplyAccNeg(
4451     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4452     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4453     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4454     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4455   assert(IdxMulOpd == 1);
4456 
4457   Register NewVR =
4458       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4459   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4460                           FMAInstKind::Accumulator, &NewVR);
4461 }
4462 
4463 /// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
4464 /// instructions.
4465 ///
4466 /// \see genFusedMultiply
4467 static MachineInstr *genFusedMultiplyIdx(
4468     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4469     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4470     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4471   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4472                           FMAInstKind::Indexed);
4473 }
4474 
4475 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4476 /// instructions with an additional negation of the accumulator
4477 static MachineInstr *genFusedMultiplyIdxNeg(
4478     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4479     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4480     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4481     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4482   assert(IdxMulOpd == 1);
4483 
4484   Register NewVR =
4485       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4486 
4487   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4488                           FMAInstKind::Indexed, &NewVR);
4489 }
4490 
4491 /// genMaddR - Generate madd instruction and combine mul and add using
4492 /// an extra virtual register
4493 /// Example - an ADD intermediate needs to be stored in a register:
4494 ///   MUL I=A,B,0
4495 ///   ADD R,I,Imm
4496 ///   ==> ORR  V, ZR, Imm
4497 ///   ==> MADD R,A,B,V
4498 /// \param MF Containing MachineFunction
4499 /// \param MRI Register information
4500 /// \param TII Target information
4501 /// \param Root is the ADD instruction
4502 /// \param [out] InsInstrs is a vector of machine instructions and will
4503 /// contain the generated madd instruction
4504 /// \param IdxMulOpd is index of operand in Root that is the result of
4505 /// the MUL. In the example above IdxMulOpd is 1.
4506 /// \param MaddOpc the opcode fo the madd instruction
4507 /// \param VR is a virtual register that holds the value of an ADD operand
4508 /// (V in the example above).
4509 /// \param RC Register class of operands
4510 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
4511                               const TargetInstrInfo *TII, MachineInstr &Root,
4512                               SmallVectorImpl<MachineInstr *> &InsInstrs,
4513                               unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
4514                               const TargetRegisterClass *RC) {
4515   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4516 
4517   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4518   Register ResultReg = Root.getOperand(0).getReg();
4519   Register SrcReg0 = MUL->getOperand(1).getReg();
4520   bool Src0IsKill = MUL->getOperand(1).isKill();
4521   Register SrcReg1 = MUL->getOperand(2).getReg();
4522   bool Src1IsKill = MUL->getOperand(2).isKill();
4523 
4524   if (Register::isVirtualRegister(ResultReg))
4525     MRI.constrainRegClass(ResultReg, RC);
4526   if (Register::isVirtualRegister(SrcReg0))
4527     MRI.constrainRegClass(SrcReg0, RC);
4528   if (Register::isVirtualRegister(SrcReg1))
4529     MRI.constrainRegClass(SrcReg1, RC);
4530   if (Register::isVirtualRegister(VR))
4531     MRI.constrainRegClass(VR, RC);
4532 
4533   MachineInstrBuilder MIB =
4534       BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4535           .addReg(SrcReg0, getKillRegState(Src0IsKill))
4536           .addReg(SrcReg1, getKillRegState(Src1IsKill))
4537           .addReg(VR);
4538   // Insert the MADD
4539   InsInstrs.push_back(MIB);
4540   return MUL;
4541 }
4542 
4543 /// When getMachineCombinerPatterns() finds potential patterns,
4544 /// this function generates the instructions that could replace the
4545 /// original code sequence
4546 void AArch64InstrInfo::genAlternativeCodeSequence(
4547     MachineInstr &Root, MachineCombinerPattern Pattern,
4548     SmallVectorImpl<MachineInstr *> &InsInstrs,
4549     SmallVectorImpl<MachineInstr *> &DelInstrs,
4550     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
4551   MachineBasicBlock &MBB = *Root.getParent();
4552   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4553   MachineFunction &MF = *MBB.getParent();
4554   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
4555 
4556   MachineInstr *MUL;
4557   const TargetRegisterClass *RC;
4558   unsigned Opc;
4559   switch (Pattern) {
4560   default:
4561     // Reassociate instructions.
4562     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
4563                                                 DelInstrs, InstrIdxForVirtReg);
4564     return;
4565   case MachineCombinerPattern::MULADDW_OP1:
4566   case MachineCombinerPattern::MULADDX_OP1:
4567     // MUL I=A,B,0
4568     // ADD R,I,C
4569     // ==> MADD R,A,B,C
4570     // --- Create(MADD);
4571     if (Pattern == MachineCombinerPattern::MULADDW_OP1) {
4572       Opc = AArch64::MADDWrrr;
4573       RC = &AArch64::GPR32RegClass;
4574     } else {
4575       Opc = AArch64::MADDXrrr;
4576       RC = &AArch64::GPR64RegClass;
4577     }
4578     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4579     break;
4580   case MachineCombinerPattern::MULADDW_OP2:
4581   case MachineCombinerPattern::MULADDX_OP2:
4582     // MUL I=A,B,0
4583     // ADD R,C,I
4584     // ==> MADD R,A,B,C
4585     // --- Create(MADD);
4586     if (Pattern == MachineCombinerPattern::MULADDW_OP2) {
4587       Opc = AArch64::MADDWrrr;
4588       RC = &AArch64::GPR32RegClass;
4589     } else {
4590       Opc = AArch64::MADDXrrr;
4591       RC = &AArch64::GPR64RegClass;
4592     }
4593     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4594     break;
4595   case MachineCombinerPattern::MULADDWI_OP1:
4596   case MachineCombinerPattern::MULADDXI_OP1: {
4597     // MUL I=A,B,0
4598     // ADD R,I,Imm
4599     // ==> ORR  V, ZR, Imm
4600     // ==> MADD R,A,B,V
4601     // --- Create(MADD);
4602     const TargetRegisterClass *OrrRC;
4603     unsigned BitSize, OrrOpc, ZeroReg;
4604     if (Pattern == MachineCombinerPattern::MULADDWI_OP1) {
4605       OrrOpc = AArch64::ORRWri;
4606       OrrRC = &AArch64::GPR32spRegClass;
4607       BitSize = 32;
4608       ZeroReg = AArch64::WZR;
4609       Opc = AArch64::MADDWrrr;
4610       RC = &AArch64::GPR32RegClass;
4611     } else {
4612       OrrOpc = AArch64::ORRXri;
4613       OrrRC = &AArch64::GPR64spRegClass;
4614       BitSize = 64;
4615       ZeroReg = AArch64::XZR;
4616       Opc = AArch64::MADDXrrr;
4617       RC = &AArch64::GPR64RegClass;
4618     }
4619     Register NewVR = MRI.createVirtualRegister(OrrRC);
4620     uint64_t Imm = Root.getOperand(2).getImm();
4621 
4622     if (Root.getOperand(3).isImm()) {
4623       unsigned Val = Root.getOperand(3).getImm();
4624       Imm = Imm << Val;
4625     }
4626     uint64_t UImm = SignExtend64(Imm, BitSize);
4627     uint64_t Encoding;
4628     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4629       MachineInstrBuilder MIB1 =
4630           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4631               .addReg(ZeroReg)
4632               .addImm(Encoding);
4633       InsInstrs.push_back(MIB1);
4634       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4635       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4636     }
4637     break;
4638   }
4639   case MachineCombinerPattern::MULSUBW_OP1:
4640   case MachineCombinerPattern::MULSUBX_OP1: {
4641     // MUL I=A,B,0
4642     // SUB R,I, C
4643     // ==> SUB  V, 0, C
4644     // ==> MADD R,A,B,V // = -C + A*B
4645     // --- Create(MADD);
4646     const TargetRegisterClass *SubRC;
4647     unsigned SubOpc, ZeroReg;
4648     if (Pattern == MachineCombinerPattern::MULSUBW_OP1) {
4649       SubOpc = AArch64::SUBWrr;
4650       SubRC = &AArch64::GPR32spRegClass;
4651       ZeroReg = AArch64::WZR;
4652       Opc = AArch64::MADDWrrr;
4653       RC = &AArch64::GPR32RegClass;
4654     } else {
4655       SubOpc = AArch64::SUBXrr;
4656       SubRC = &AArch64::GPR64spRegClass;
4657       ZeroReg = AArch64::XZR;
4658       Opc = AArch64::MADDXrrr;
4659       RC = &AArch64::GPR64RegClass;
4660     }
4661     Register NewVR = MRI.createVirtualRegister(SubRC);
4662     // SUB NewVR, 0, C
4663     MachineInstrBuilder MIB1 =
4664         BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
4665             .addReg(ZeroReg)
4666             .add(Root.getOperand(2));
4667     InsInstrs.push_back(MIB1);
4668     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4669     MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4670     break;
4671   }
4672   case MachineCombinerPattern::MULSUBW_OP2:
4673   case MachineCombinerPattern::MULSUBX_OP2:
4674     // MUL I=A,B,0
4675     // SUB R,C,I
4676     // ==> MSUB R,A,B,C (computes C - A*B)
4677     // --- Create(MSUB);
4678     if (Pattern == MachineCombinerPattern::MULSUBW_OP2) {
4679       Opc = AArch64::MSUBWrrr;
4680       RC = &AArch64::GPR32RegClass;
4681     } else {
4682       Opc = AArch64::MSUBXrrr;
4683       RC = &AArch64::GPR64RegClass;
4684     }
4685     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4686     break;
4687   case MachineCombinerPattern::MULSUBWI_OP1:
4688   case MachineCombinerPattern::MULSUBXI_OP1: {
4689     // MUL I=A,B,0
4690     // SUB R,I, Imm
4691     // ==> ORR  V, ZR, -Imm
4692     // ==> MADD R,A,B,V // = -Imm + A*B
4693     // --- Create(MADD);
4694     const TargetRegisterClass *OrrRC;
4695     unsigned BitSize, OrrOpc, ZeroReg;
4696     if (Pattern == MachineCombinerPattern::MULSUBWI_OP1) {
4697       OrrOpc = AArch64::ORRWri;
4698       OrrRC = &AArch64::GPR32spRegClass;
4699       BitSize = 32;
4700       ZeroReg = AArch64::WZR;
4701       Opc = AArch64::MADDWrrr;
4702       RC = &AArch64::GPR32RegClass;
4703     } else {
4704       OrrOpc = AArch64::ORRXri;
4705       OrrRC = &AArch64::GPR64spRegClass;
4706       BitSize = 64;
4707       ZeroReg = AArch64::XZR;
4708       Opc = AArch64::MADDXrrr;
4709       RC = &AArch64::GPR64RegClass;
4710     }
4711     Register NewVR = MRI.createVirtualRegister(OrrRC);
4712     uint64_t Imm = Root.getOperand(2).getImm();
4713     if (Root.getOperand(3).isImm()) {
4714       unsigned Val = Root.getOperand(3).getImm();
4715       Imm = Imm << Val;
4716     }
4717     uint64_t UImm = SignExtend64(-Imm, BitSize);
4718     uint64_t Encoding;
4719     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4720       MachineInstrBuilder MIB1 =
4721           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4722               .addReg(ZeroReg)
4723               .addImm(Encoding);
4724       InsInstrs.push_back(MIB1);
4725       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4726       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4727     }
4728     break;
4729   }
4730 
4731   case MachineCombinerPattern::MULADDv8i8_OP1:
4732     Opc = AArch64::MLAv8i8;
4733     RC = &AArch64::FPR64RegClass;
4734     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4735     break;
4736   case MachineCombinerPattern::MULADDv8i8_OP2:
4737     Opc = AArch64::MLAv8i8;
4738     RC = &AArch64::FPR64RegClass;
4739     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4740     break;
4741   case MachineCombinerPattern::MULADDv16i8_OP1:
4742     Opc = AArch64::MLAv16i8;
4743     RC = &AArch64::FPR128RegClass;
4744     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4745     break;
4746   case MachineCombinerPattern::MULADDv16i8_OP2:
4747     Opc = AArch64::MLAv16i8;
4748     RC = &AArch64::FPR128RegClass;
4749     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4750     break;
4751   case MachineCombinerPattern::MULADDv4i16_OP1:
4752     Opc = AArch64::MLAv4i16;
4753     RC = &AArch64::FPR64RegClass;
4754     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4755     break;
4756   case MachineCombinerPattern::MULADDv4i16_OP2:
4757     Opc = AArch64::MLAv4i16;
4758     RC = &AArch64::FPR64RegClass;
4759     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4760     break;
4761   case MachineCombinerPattern::MULADDv8i16_OP1:
4762     Opc = AArch64::MLAv8i16;
4763     RC = &AArch64::FPR128RegClass;
4764     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4765     break;
4766   case MachineCombinerPattern::MULADDv8i16_OP2:
4767     Opc = AArch64::MLAv8i16;
4768     RC = &AArch64::FPR128RegClass;
4769     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4770     break;
4771   case MachineCombinerPattern::MULADDv2i32_OP1:
4772     Opc = AArch64::MLAv2i32;
4773     RC = &AArch64::FPR64RegClass;
4774     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4775     break;
4776   case MachineCombinerPattern::MULADDv2i32_OP2:
4777     Opc = AArch64::MLAv2i32;
4778     RC = &AArch64::FPR64RegClass;
4779     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4780     break;
4781   case MachineCombinerPattern::MULADDv4i32_OP1:
4782     Opc = AArch64::MLAv4i32;
4783     RC = &AArch64::FPR128RegClass;
4784     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4785     break;
4786   case MachineCombinerPattern::MULADDv4i32_OP2:
4787     Opc = AArch64::MLAv4i32;
4788     RC = &AArch64::FPR128RegClass;
4789     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4790     break;
4791 
4792   case MachineCombinerPattern::MULSUBv8i8_OP1:
4793     Opc = AArch64::MLAv8i8;
4794     RC = &AArch64::FPR64RegClass;
4795     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4796                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
4797                                  RC);
4798     break;
4799   case MachineCombinerPattern::MULSUBv8i8_OP2:
4800     Opc = AArch64::MLSv8i8;
4801     RC = &AArch64::FPR64RegClass;
4802     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4803     break;
4804   case MachineCombinerPattern::MULSUBv16i8_OP1:
4805     Opc = AArch64::MLAv16i8;
4806     RC = &AArch64::FPR128RegClass;
4807     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4808                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
4809                                  RC);
4810     break;
4811   case MachineCombinerPattern::MULSUBv16i8_OP2:
4812     Opc = AArch64::MLSv16i8;
4813     RC = &AArch64::FPR128RegClass;
4814     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4815     break;
4816   case MachineCombinerPattern::MULSUBv4i16_OP1:
4817     Opc = AArch64::MLAv4i16;
4818     RC = &AArch64::FPR64RegClass;
4819     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4820                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4821                                  RC);
4822     break;
4823   case MachineCombinerPattern::MULSUBv4i16_OP2:
4824     Opc = AArch64::MLSv4i16;
4825     RC = &AArch64::FPR64RegClass;
4826     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4827     break;
4828   case MachineCombinerPattern::MULSUBv8i16_OP1:
4829     Opc = AArch64::MLAv8i16;
4830     RC = &AArch64::FPR128RegClass;
4831     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4832                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4833                                  RC);
4834     break;
4835   case MachineCombinerPattern::MULSUBv8i16_OP2:
4836     Opc = AArch64::MLSv8i16;
4837     RC = &AArch64::FPR128RegClass;
4838     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4839     break;
4840   case MachineCombinerPattern::MULSUBv2i32_OP1:
4841     Opc = AArch64::MLAv2i32;
4842     RC = &AArch64::FPR64RegClass;
4843     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4844                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4845                                  RC);
4846     break;
4847   case MachineCombinerPattern::MULSUBv2i32_OP2:
4848     Opc = AArch64::MLSv2i32;
4849     RC = &AArch64::FPR64RegClass;
4850     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4851     break;
4852   case MachineCombinerPattern::MULSUBv4i32_OP1:
4853     Opc = AArch64::MLAv4i32;
4854     RC = &AArch64::FPR128RegClass;
4855     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4856                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
4857                                  RC);
4858     break;
4859   case MachineCombinerPattern::MULSUBv4i32_OP2:
4860     Opc = AArch64::MLSv4i32;
4861     RC = &AArch64::FPR128RegClass;
4862     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4863     break;
4864 
4865   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4866     Opc = AArch64::MLAv4i16_indexed;
4867     RC = &AArch64::FPR64RegClass;
4868     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4869     break;
4870   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4871     Opc = AArch64::MLAv4i16_indexed;
4872     RC = &AArch64::FPR64RegClass;
4873     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4874     break;
4875   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4876     Opc = AArch64::MLAv8i16_indexed;
4877     RC = &AArch64::FPR128RegClass;
4878     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4879     break;
4880   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4881     Opc = AArch64::MLAv8i16_indexed;
4882     RC = &AArch64::FPR128RegClass;
4883     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4884     break;
4885   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4886     Opc = AArch64::MLAv2i32_indexed;
4887     RC = &AArch64::FPR64RegClass;
4888     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4889     break;
4890   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4891     Opc = AArch64::MLAv2i32_indexed;
4892     RC = &AArch64::FPR64RegClass;
4893     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4894     break;
4895   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4896     Opc = AArch64::MLAv4i32_indexed;
4897     RC = &AArch64::FPR128RegClass;
4898     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4899     break;
4900   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4901     Opc = AArch64::MLAv4i32_indexed;
4902     RC = &AArch64::FPR128RegClass;
4903     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4904     break;
4905 
4906   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4907     Opc = AArch64::MLAv4i16_indexed;
4908     RC = &AArch64::FPR64RegClass;
4909     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4910                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4911                                  RC);
4912     break;
4913   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4914     Opc = AArch64::MLSv4i16_indexed;
4915     RC = &AArch64::FPR64RegClass;
4916     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4917     break;
4918   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4919     Opc = AArch64::MLAv8i16_indexed;
4920     RC = &AArch64::FPR128RegClass;
4921     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4922                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4923                                  RC);
4924     break;
4925   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4926     Opc = AArch64::MLSv8i16_indexed;
4927     RC = &AArch64::FPR128RegClass;
4928     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4929     break;
4930   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4931     Opc = AArch64::MLAv2i32_indexed;
4932     RC = &AArch64::FPR64RegClass;
4933     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4934                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4935                                  RC);
4936     break;
4937   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4938     Opc = AArch64::MLSv2i32_indexed;
4939     RC = &AArch64::FPR64RegClass;
4940     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4941     break;
4942   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4943     Opc = AArch64::MLAv4i32_indexed;
4944     RC = &AArch64::FPR128RegClass;
4945     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4946                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
4947                                  RC);
4948     break;
4949   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4950     Opc = AArch64::MLSv4i32_indexed;
4951     RC = &AArch64::FPR128RegClass;
4952     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4953     break;
4954 
4955   // Floating Point Support
4956   case MachineCombinerPattern::FMULADDH_OP1:
4957     Opc = AArch64::FMADDHrrr;
4958     RC = &AArch64::FPR16RegClass;
4959     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4960     break;
4961   case MachineCombinerPattern::FMULADDS_OP1:
4962     Opc = AArch64::FMADDSrrr;
4963     RC = &AArch64::FPR32RegClass;
4964     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4965     break;
4966   case MachineCombinerPattern::FMULADDD_OP1:
4967     Opc = AArch64::FMADDDrrr;
4968     RC = &AArch64::FPR64RegClass;
4969     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4970     break;
4971 
4972   case MachineCombinerPattern::FMULADDH_OP2:
4973     Opc = AArch64::FMADDHrrr;
4974     RC = &AArch64::FPR16RegClass;
4975     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4976     break;
4977   case MachineCombinerPattern::FMULADDS_OP2:
4978     Opc = AArch64::FMADDSrrr;
4979     RC = &AArch64::FPR32RegClass;
4980     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4981     break;
4982   case MachineCombinerPattern::FMULADDD_OP2:
4983     Opc = AArch64::FMADDDrrr;
4984     RC = &AArch64::FPR64RegClass;
4985     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4986     break;
4987 
4988   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4989     Opc = AArch64::FMLAv1i32_indexed;
4990     RC = &AArch64::FPR32RegClass;
4991     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4992                            FMAInstKind::Indexed);
4993     break;
4994   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4995     Opc = AArch64::FMLAv1i32_indexed;
4996     RC = &AArch64::FPR32RegClass;
4997     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4998                            FMAInstKind::Indexed);
4999     break;
5000 
5001   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
5002     Opc = AArch64::FMLAv1i64_indexed;
5003     RC = &AArch64::FPR64RegClass;
5004     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5005                            FMAInstKind::Indexed);
5006     break;
5007   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
5008     Opc = AArch64::FMLAv1i64_indexed;
5009     RC = &AArch64::FPR64RegClass;
5010     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5011                            FMAInstKind::Indexed);
5012     break;
5013 
5014   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
5015     RC = &AArch64::FPR64RegClass;
5016     Opc = AArch64::FMLAv4i16_indexed;
5017     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5018                            FMAInstKind::Indexed);
5019     break;
5020   case MachineCombinerPattern::FMLAv4f16_OP1:
5021     RC = &AArch64::FPR64RegClass;
5022     Opc = AArch64::FMLAv4f16;
5023     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5024                            FMAInstKind::Accumulator);
5025     break;
5026   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
5027     RC = &AArch64::FPR64RegClass;
5028     Opc = AArch64::FMLAv4i16_indexed;
5029     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5030                            FMAInstKind::Indexed);
5031     break;
5032   case MachineCombinerPattern::FMLAv4f16_OP2:
5033     RC = &AArch64::FPR64RegClass;
5034     Opc = AArch64::FMLAv4f16;
5035     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5036                            FMAInstKind::Accumulator);
5037     break;
5038 
5039   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
5040   case MachineCombinerPattern::FMLAv2f32_OP1:
5041     RC = &AArch64::FPR64RegClass;
5042     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
5043       Opc = AArch64::FMLAv2i32_indexed;
5044       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5045                              FMAInstKind::Indexed);
5046     } else {
5047       Opc = AArch64::FMLAv2f32;
5048       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5049                              FMAInstKind::Accumulator);
5050     }
5051     break;
5052   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
5053   case MachineCombinerPattern::FMLAv2f32_OP2:
5054     RC = &AArch64::FPR64RegClass;
5055     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
5056       Opc = AArch64::FMLAv2i32_indexed;
5057       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5058                              FMAInstKind::Indexed);
5059     } else {
5060       Opc = AArch64::FMLAv2f32;
5061       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5062                              FMAInstKind::Accumulator);
5063     }
5064     break;
5065 
5066   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
5067     RC = &AArch64::FPR128RegClass;
5068     Opc = AArch64::FMLAv8i16_indexed;
5069     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5070                            FMAInstKind::Indexed);
5071     break;
5072   case MachineCombinerPattern::FMLAv8f16_OP1:
5073     RC = &AArch64::FPR128RegClass;
5074     Opc = AArch64::FMLAv8f16;
5075     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5076                            FMAInstKind::Accumulator);
5077     break;
5078   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
5079     RC = &AArch64::FPR128RegClass;
5080     Opc = AArch64::FMLAv8i16_indexed;
5081     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5082                            FMAInstKind::Indexed);
5083     break;
5084   case MachineCombinerPattern::FMLAv8f16_OP2:
5085     RC = &AArch64::FPR128RegClass;
5086     Opc = AArch64::FMLAv8f16;
5087     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5088                            FMAInstKind::Accumulator);
5089     break;
5090 
5091   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
5092   case MachineCombinerPattern::FMLAv2f64_OP1:
5093     RC = &AArch64::FPR128RegClass;
5094     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
5095       Opc = AArch64::FMLAv2i64_indexed;
5096       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5097                              FMAInstKind::Indexed);
5098     } else {
5099       Opc = AArch64::FMLAv2f64;
5100       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5101                              FMAInstKind::Accumulator);
5102     }
5103     break;
5104   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
5105   case MachineCombinerPattern::FMLAv2f64_OP2:
5106     RC = &AArch64::FPR128RegClass;
5107     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
5108       Opc = AArch64::FMLAv2i64_indexed;
5109       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5110                              FMAInstKind::Indexed);
5111     } else {
5112       Opc = AArch64::FMLAv2f64;
5113       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5114                              FMAInstKind::Accumulator);
5115     }
5116     break;
5117 
5118   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
5119   case MachineCombinerPattern::FMLAv4f32_OP1:
5120     RC = &AArch64::FPR128RegClass;
5121     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
5122       Opc = AArch64::FMLAv4i32_indexed;
5123       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5124                              FMAInstKind::Indexed);
5125     } else {
5126       Opc = AArch64::FMLAv4f32;
5127       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5128                              FMAInstKind::Accumulator);
5129     }
5130     break;
5131 
5132   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
5133   case MachineCombinerPattern::FMLAv4f32_OP2:
5134     RC = &AArch64::FPR128RegClass;
5135     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
5136       Opc = AArch64::FMLAv4i32_indexed;
5137       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5138                              FMAInstKind::Indexed);
5139     } else {
5140       Opc = AArch64::FMLAv4f32;
5141       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5142                              FMAInstKind::Accumulator);
5143     }
5144     break;
5145 
5146   case MachineCombinerPattern::FMULSUBH_OP1:
5147     Opc = AArch64::FNMSUBHrrr;
5148     RC = &AArch64::FPR16RegClass;
5149     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5150     break;
5151   case MachineCombinerPattern::FMULSUBS_OP1:
5152     Opc = AArch64::FNMSUBSrrr;
5153     RC = &AArch64::FPR32RegClass;
5154     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5155     break;
5156   case MachineCombinerPattern::FMULSUBD_OP1:
5157     Opc = AArch64::FNMSUBDrrr;
5158     RC = &AArch64::FPR64RegClass;
5159     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5160     break;
5161 
5162   case MachineCombinerPattern::FNMULSUBH_OP1:
5163     Opc = AArch64::FNMADDHrrr;
5164     RC = &AArch64::FPR16RegClass;
5165     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5166     break;
5167   case MachineCombinerPattern::FNMULSUBS_OP1:
5168     Opc = AArch64::FNMADDSrrr;
5169     RC = &AArch64::FPR32RegClass;
5170     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5171     break;
5172   case MachineCombinerPattern::FNMULSUBD_OP1:
5173     Opc = AArch64::FNMADDDrrr;
5174     RC = &AArch64::FPR64RegClass;
5175     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5176     break;
5177 
5178   case MachineCombinerPattern::FMULSUBH_OP2:
5179     Opc = AArch64::FMSUBHrrr;
5180     RC = &AArch64::FPR16RegClass;
5181     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5182     break;
5183   case MachineCombinerPattern::FMULSUBS_OP2:
5184     Opc = AArch64::FMSUBSrrr;
5185     RC = &AArch64::FPR32RegClass;
5186     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5187     break;
5188   case MachineCombinerPattern::FMULSUBD_OP2:
5189     Opc = AArch64::FMSUBDrrr;
5190     RC = &AArch64::FPR64RegClass;
5191     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5192     break;
5193 
5194   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
5195     Opc = AArch64::FMLSv1i32_indexed;
5196     RC = &AArch64::FPR32RegClass;
5197     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5198                            FMAInstKind::Indexed);
5199     break;
5200 
5201   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
5202     Opc = AArch64::FMLSv1i64_indexed;
5203     RC = &AArch64::FPR64RegClass;
5204     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5205                            FMAInstKind::Indexed);
5206     break;
5207 
5208   case MachineCombinerPattern::FMLSv4f16_OP1:
5209   case MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
5210     RC = &AArch64::FPR64RegClass;
5211     Register NewVR = MRI.createVirtualRegister(RC);
5212     MachineInstrBuilder MIB1 =
5213         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f16), NewVR)
5214             .add(Root.getOperand(2));
5215     InsInstrs.push_back(MIB1);
5216     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5217     if (Pattern == MachineCombinerPattern::FMLSv4f16_OP1) {
5218       Opc = AArch64::FMLAv4f16;
5219       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5220                              FMAInstKind::Accumulator, &NewVR);
5221     } else {
5222       Opc = AArch64::FMLAv4i16_indexed;
5223       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5224                              FMAInstKind::Indexed, &NewVR);
5225     }
5226     break;
5227   }
5228   case MachineCombinerPattern::FMLSv4f16_OP2:
5229     RC = &AArch64::FPR64RegClass;
5230     Opc = AArch64::FMLSv4f16;
5231     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5232                            FMAInstKind::Accumulator);
5233     break;
5234   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
5235     RC = &AArch64::FPR64RegClass;
5236     Opc = AArch64::FMLSv4i16_indexed;
5237     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5238                            FMAInstKind::Indexed);
5239     break;
5240 
5241   case MachineCombinerPattern::FMLSv2f32_OP2:
5242   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
5243     RC = &AArch64::FPR64RegClass;
5244     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
5245       Opc = AArch64::FMLSv2i32_indexed;
5246       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5247                              FMAInstKind::Indexed);
5248     } else {
5249       Opc = AArch64::FMLSv2f32;
5250       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5251                              FMAInstKind::Accumulator);
5252     }
5253     break;
5254 
5255   case MachineCombinerPattern::FMLSv8f16_OP1:
5256   case MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
5257     RC = &AArch64::FPR128RegClass;
5258     Register NewVR = MRI.createVirtualRegister(RC);
5259     MachineInstrBuilder MIB1 =
5260         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv8f16), NewVR)
5261             .add(Root.getOperand(2));
5262     InsInstrs.push_back(MIB1);
5263     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5264     if (Pattern == MachineCombinerPattern::FMLSv8f16_OP1) {
5265       Opc = AArch64::FMLAv8f16;
5266       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5267                              FMAInstKind::Accumulator, &NewVR);
5268     } else {
5269       Opc = AArch64::FMLAv8i16_indexed;
5270       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5271                              FMAInstKind::Indexed, &NewVR);
5272     }
5273     break;
5274   }
5275   case MachineCombinerPattern::FMLSv8f16_OP2:
5276     RC = &AArch64::FPR128RegClass;
5277     Opc = AArch64::FMLSv8f16;
5278     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5279                            FMAInstKind::Accumulator);
5280     break;
5281   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
5282     RC = &AArch64::FPR128RegClass;
5283     Opc = AArch64::FMLSv8i16_indexed;
5284     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5285                            FMAInstKind::Indexed);
5286     break;
5287 
5288   case MachineCombinerPattern::FMLSv2f64_OP2:
5289   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
5290     RC = &AArch64::FPR128RegClass;
5291     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
5292       Opc = AArch64::FMLSv2i64_indexed;
5293       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5294                              FMAInstKind::Indexed);
5295     } else {
5296       Opc = AArch64::FMLSv2f64;
5297       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5298                              FMAInstKind::Accumulator);
5299     }
5300     break;
5301 
5302   case MachineCombinerPattern::FMLSv4f32_OP2:
5303   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
5304     RC = &AArch64::FPR128RegClass;
5305     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
5306       Opc = AArch64::FMLSv4i32_indexed;
5307       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5308                              FMAInstKind::Indexed);
5309     } else {
5310       Opc = AArch64::FMLSv4f32;
5311       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5312                              FMAInstKind::Accumulator);
5313     }
5314     break;
5315   case MachineCombinerPattern::FMLSv2f32_OP1:
5316   case MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
5317     RC = &AArch64::FPR64RegClass;
5318     Register NewVR = MRI.createVirtualRegister(RC);
5319     MachineInstrBuilder MIB1 =
5320         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f32), NewVR)
5321             .add(Root.getOperand(2));
5322     InsInstrs.push_back(MIB1);
5323     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5324     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
5325       Opc = AArch64::FMLAv2i32_indexed;
5326       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5327                              FMAInstKind::Indexed, &NewVR);
5328     } else {
5329       Opc = AArch64::FMLAv2f32;
5330       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5331                              FMAInstKind::Accumulator, &NewVR);
5332     }
5333     break;
5334   }
5335   case MachineCombinerPattern::FMLSv4f32_OP1:
5336   case MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
5337     RC = &AArch64::FPR128RegClass;
5338     Register NewVR = MRI.createVirtualRegister(RC);
5339     MachineInstrBuilder MIB1 =
5340         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f32), NewVR)
5341             .add(Root.getOperand(2));
5342     InsInstrs.push_back(MIB1);
5343     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5344     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
5345       Opc = AArch64::FMLAv4i32_indexed;
5346       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5347                              FMAInstKind::Indexed, &NewVR);
5348     } else {
5349       Opc = AArch64::FMLAv4f32;
5350       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5351                              FMAInstKind::Accumulator, &NewVR);
5352     }
5353     break;
5354   }
5355   case MachineCombinerPattern::FMLSv2f64_OP1:
5356   case MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
5357     RC = &AArch64::FPR128RegClass;
5358     Register NewVR = MRI.createVirtualRegister(RC);
5359     MachineInstrBuilder MIB1 =
5360         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f64), NewVR)
5361             .add(Root.getOperand(2));
5362     InsInstrs.push_back(MIB1);
5363     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5364     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
5365       Opc = AArch64::FMLAv2i64_indexed;
5366       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5367                              FMAInstKind::Indexed, &NewVR);
5368     } else {
5369       Opc = AArch64::FMLAv2f64;
5370       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5371                              FMAInstKind::Accumulator, &NewVR);
5372     }
5373     break;
5374   }
5375   } // end switch (Pattern)
5376   // Record MUL and ADD/SUB for deletion
5377   DelInstrs.push_back(MUL);
5378   DelInstrs.push_back(&Root);
5379 }
5380 
5381 /// Replace csincr-branch sequence by simple conditional branch
5382 ///
5383 /// Examples:
5384 /// 1. \code
5385 ///   csinc  w9, wzr, wzr, <condition code>
5386 ///   tbnz   w9, #0, 0x44
5387 ///    \endcode
5388 /// to
5389 ///    \code
5390 ///   b.<inverted condition code>
5391 ///    \endcode
5392 ///
5393 /// 2. \code
5394 ///   csinc w9, wzr, wzr, <condition code>
5395 ///   tbz   w9, #0, 0x44
5396 ///    \endcode
5397 /// to
5398 ///    \code
5399 ///   b.<condition code>
5400 ///    \endcode
5401 ///
5402 /// Replace compare and branch sequence by TBZ/TBNZ instruction when the
5403 /// compare's constant operand is power of 2.
5404 ///
5405 /// Examples:
5406 ///    \code
5407 ///   and  w8, w8, #0x400
5408 ///   cbnz w8, L1
5409 ///    \endcode
5410 /// to
5411 ///    \code
5412 ///   tbnz w8, #10, L1
5413 ///    \endcode
5414 ///
5415 /// \param  MI Conditional Branch
5416 /// \return True when the simple conditional branch is generated
5417 ///
5418 bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
5419   bool IsNegativeBranch = false;
5420   bool IsTestAndBranch = false;
5421   unsigned TargetBBInMI = 0;
5422   switch (MI.getOpcode()) {
5423   default:
5424     llvm_unreachable("Unknown branch instruction?");
5425   case AArch64::Bcc:
5426     return false;
5427   case AArch64::CBZW:
5428   case AArch64::CBZX:
5429     TargetBBInMI = 1;
5430     break;
5431   case AArch64::CBNZW:
5432   case AArch64::CBNZX:
5433     TargetBBInMI = 1;
5434     IsNegativeBranch = true;
5435     break;
5436   case AArch64::TBZW:
5437   case AArch64::TBZX:
5438     TargetBBInMI = 2;
5439     IsTestAndBranch = true;
5440     break;
5441   case AArch64::TBNZW:
5442   case AArch64::TBNZX:
5443     TargetBBInMI = 2;
5444     IsNegativeBranch = true;
5445     IsTestAndBranch = true;
5446     break;
5447   }
5448   // So we increment a zero register and test for bits other
5449   // than bit 0? Conservatively bail out in case the verifier
5450   // missed this case.
5451   if (IsTestAndBranch && MI.getOperand(1).getImm())
5452     return false;
5453 
5454   // Find Definition.
5455   assert(MI.getParent() && "Incomplete machine instruciton\n");
5456   MachineBasicBlock *MBB = MI.getParent();
5457   MachineFunction *MF = MBB->getParent();
5458   MachineRegisterInfo *MRI = &MF->getRegInfo();
5459   Register VReg = MI.getOperand(0).getReg();
5460   if (!Register::isVirtualRegister(VReg))
5461     return false;
5462 
5463   MachineInstr *DefMI = MRI->getVRegDef(VReg);
5464 
5465   // Look through COPY instructions to find definition.
5466   while (DefMI->isCopy()) {
5467     Register CopyVReg = DefMI->getOperand(1).getReg();
5468     if (!MRI->hasOneNonDBGUse(CopyVReg))
5469       return false;
5470     if (!MRI->hasOneDef(CopyVReg))
5471       return false;
5472     DefMI = MRI->getVRegDef(CopyVReg);
5473   }
5474 
5475   switch (DefMI->getOpcode()) {
5476   default:
5477     return false;
5478   // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
5479   case AArch64::ANDWri:
5480   case AArch64::ANDXri: {
5481     if (IsTestAndBranch)
5482       return false;
5483     if (DefMI->getParent() != MBB)
5484       return false;
5485     if (!MRI->hasOneNonDBGUse(VReg))
5486       return false;
5487 
5488     bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
5489     uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
5490         DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
5491     if (!isPowerOf2_64(Mask))
5492       return false;
5493 
5494     MachineOperand &MO = DefMI->getOperand(1);
5495     Register NewReg = MO.getReg();
5496     if (!Register::isVirtualRegister(NewReg))
5497       return false;
5498 
5499     assert(!MRI->def_empty(NewReg) && "Register must be defined.");
5500 
5501     MachineBasicBlock &RefToMBB = *MBB;
5502     MachineBasicBlock *TBB = MI.getOperand(1).getMBB();
5503     DebugLoc DL = MI.getDebugLoc();
5504     unsigned Imm = Log2_64(Mask);
5505     unsigned Opc = (Imm < 32)
5506                        ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
5507                        : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
5508     MachineInstr *NewMI = BuildMI(RefToMBB, MI, DL, get(Opc))
5509                               .addReg(NewReg)
5510                               .addImm(Imm)
5511                               .addMBB(TBB);
5512     // Register lives on to the CBZ now.
5513     MO.setIsKill(false);
5514 
5515     // For immediate smaller than 32, we need to use the 32-bit
5516     // variant (W) in all cases. Indeed the 64-bit variant does not
5517     // allow to encode them.
5518     // Therefore, if the input register is 64-bit, we need to take the
5519     // 32-bit sub-part.
5520     if (!Is32Bit && Imm < 32)
5521       NewMI->getOperand(0).setSubReg(AArch64::sub_32);
5522     MI.eraseFromParent();
5523     return true;
5524   }
5525   // Look for CSINC
5526   case AArch64::CSINCWr:
5527   case AArch64::CSINCXr: {
5528     if (!(DefMI->getOperand(1).getReg() == AArch64::WZR &&
5529           DefMI->getOperand(2).getReg() == AArch64::WZR) &&
5530         !(DefMI->getOperand(1).getReg() == AArch64::XZR &&
5531           DefMI->getOperand(2).getReg() == AArch64::XZR))
5532       return false;
5533 
5534     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
5535       return false;
5536 
5537     AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(3).getImm();
5538     // Convert only when the condition code is not modified between
5539     // the CSINC and the branch. The CC may be used by other
5540     // instructions in between.
5541     if (areCFlagsAccessedBetweenInstrs(DefMI, MI, &getRegisterInfo(), AK_Write))
5542       return false;
5543     MachineBasicBlock &RefToMBB = *MBB;
5544     MachineBasicBlock *TBB = MI.getOperand(TargetBBInMI).getMBB();
5545     DebugLoc DL = MI.getDebugLoc();
5546     if (IsNegativeBranch)
5547       CC = AArch64CC::getInvertedCondCode(CC);
5548     BuildMI(RefToMBB, MI, DL, get(AArch64::Bcc)).addImm(CC).addMBB(TBB);
5549     MI.eraseFromParent();
5550     return true;
5551   }
5552   }
5553 }
5554 
5555 std::pair<unsigned, unsigned>
5556 AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5557   const unsigned Mask = AArch64II::MO_FRAGMENT;
5558   return std::make_pair(TF & Mask, TF & ~Mask);
5559 }
5560 
5561 ArrayRef<std::pair<unsigned, const char *>>
5562 AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5563   using namespace AArch64II;
5564 
5565   static const std::pair<unsigned, const char *> TargetFlags[] = {
5566       {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
5567       {MO_G3, "aarch64-g3"},     {MO_G2, "aarch64-g2"},
5568       {MO_G1, "aarch64-g1"},     {MO_G0, "aarch64-g0"},
5569       {MO_HI12, "aarch64-hi12"}};
5570   return makeArrayRef(TargetFlags);
5571 }
5572 
5573 ArrayRef<std::pair<unsigned, const char *>>
5574 AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5575   using namespace AArch64II;
5576 
5577   static const std::pair<unsigned, const char *> TargetFlags[] = {
5578       {MO_COFFSTUB, "aarch64-coffstub"},
5579       {MO_GOT, "aarch64-got"},
5580       {MO_NC, "aarch64-nc"},
5581       {MO_S, "aarch64-s"},
5582       {MO_TLS, "aarch64-tls"},
5583       {MO_DLLIMPORT, "aarch64-dllimport"},
5584       {MO_PREL, "aarch64-prel"},
5585       {MO_TAGGED, "aarch64-tagged"}};
5586   return makeArrayRef(TargetFlags);
5587 }
5588 
5589 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5590 AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5591   static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5592       {{MOSuppressPair, "aarch64-suppress-pair"},
5593        {MOStridedAccess, "aarch64-strided-access"}};
5594   return makeArrayRef(TargetFlags);
5595 }
5596 
5597 /// Constants defining how certain sequences should be outlined.
5598 /// This encompasses how an outlined function should be called, and what kind of
5599 /// frame should be emitted for that outlined function.
5600 ///
5601 /// \p MachineOutlinerDefault implies that the function should be called with
5602 /// a save and restore of LR to the stack.
5603 ///
5604 /// That is,
5605 ///
5606 /// I1     Save LR                    OUTLINED_FUNCTION:
5607 /// I2 --> BL OUTLINED_FUNCTION       I1
5608 /// I3     Restore LR                 I2
5609 ///                                   I3
5610 ///                                   RET
5611 ///
5612 /// * Call construction overhead: 3 (save + BL + restore)
5613 /// * Frame construction overhead: 1 (ret)
5614 /// * Requires stack fixups? Yes
5615 ///
5616 /// \p MachineOutlinerTailCall implies that the function is being created from
5617 /// a sequence of instructions ending in a return.
5618 ///
5619 /// That is,
5620 ///
5621 /// I1                             OUTLINED_FUNCTION:
5622 /// I2 --> B OUTLINED_FUNCTION     I1
5623 /// RET                            I2
5624 ///                                RET
5625 ///
5626 /// * Call construction overhead: 1 (B)
5627 /// * Frame construction overhead: 0 (Return included in sequence)
5628 /// * Requires stack fixups? No
5629 ///
5630 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5631 /// a BL instruction, but doesn't require LR to be saved and restored. This
5632 /// happens when LR is known to be dead.
5633 ///
5634 /// That is,
5635 ///
5636 /// I1                                OUTLINED_FUNCTION:
5637 /// I2 --> BL OUTLINED_FUNCTION       I1
5638 /// I3                                I2
5639 ///                                   I3
5640 ///                                   RET
5641 ///
5642 /// * Call construction overhead: 1 (BL)
5643 /// * Frame construction overhead: 1 (RET)
5644 /// * Requires stack fixups? No
5645 ///
5646 /// \p MachineOutlinerThunk implies that the function is being created from
5647 /// a sequence of instructions ending in a call. The outlined function is
5648 /// called with a BL instruction, and the outlined function tail-calls the
5649 /// original call destination.
5650 ///
5651 /// That is,
5652 ///
5653 /// I1                                OUTLINED_FUNCTION:
5654 /// I2 --> BL OUTLINED_FUNCTION       I1
5655 /// BL f                              I2
5656 ///                                   B f
5657 /// * Call construction overhead: 1 (BL)
5658 /// * Frame construction overhead: 0
5659 /// * Requires stack fixups? No
5660 ///
5661 /// \p MachineOutlinerRegSave implies that the function should be called with a
5662 /// save and restore of LR to an available register. This allows us to avoid
5663 /// stack fixups. Note that this outlining variant is compatible with the
5664 /// NoLRSave case.
5665 ///
5666 /// That is,
5667 ///
5668 /// I1     Save LR                    OUTLINED_FUNCTION:
5669 /// I2 --> BL OUTLINED_FUNCTION       I1
5670 /// I3     Restore LR                 I2
5671 ///                                   I3
5672 ///                                   RET
5673 ///
5674 /// * Call construction overhead: 3 (save + BL + restore)
5675 /// * Frame construction overhead: 1 (ret)
5676 /// * Requires stack fixups? No
5677 enum MachineOutlinerClass {
5678   MachineOutlinerDefault,  /// Emit a save, restore, call, and return.
5679   MachineOutlinerTailCall, /// Only emit a branch.
5680   MachineOutlinerNoLRSave, /// Emit a call and return.
5681   MachineOutlinerThunk,    /// Emit a call and tail-call.
5682   MachineOutlinerRegSave   /// Same as default, but save to a register.
5683 };
5684 
5685 enum MachineOutlinerMBBFlags {
5686   LRUnavailableSomewhere = 0x2,
5687   HasCalls = 0x4,
5688   UnsafeRegsDead = 0x8
5689 };
5690 
5691 unsigned
5692 AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5693   assert(C.LRUWasSet && "LRU wasn't set?");
5694   MachineFunction *MF = C.getMF();
5695   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
5696       MF->getSubtarget().getRegisterInfo());
5697 
5698   // Check if there is an available register across the sequence that we can
5699   // use.
5700   for (unsigned Reg : AArch64::GPR64RegClass) {
5701     if (!ARI->isReservedReg(*MF, Reg) &&
5702         Reg != AArch64::LR &&  // LR is not reserved, but don't use it.
5703         Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
5704         Reg != AArch64::X17 && // Ditto for X17.
5705         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5706       return Reg;
5707   }
5708 
5709   // No suitable register. Return 0.
5710   return 0u;
5711 }
5712 
5713 static bool
5714 outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
5715                                          const outliner::Candidate &b) {
5716   const Function &Fa = a.getMF()->getFunction();
5717   const Function &Fb = b.getMF()->getFunction();
5718 
5719   // If none of the functions have the "sign-return-address" attribute their
5720   // signing behaviour is equal
5721   if (!Fa.hasFnAttribute("sign-return-address") &&
5722       !Fb.hasFnAttribute("sign-return-address")) {
5723     return true;
5724   }
5725 
5726   // If both functions have the "sign-return-address" attribute their signing
5727   // behaviour is equal, if the values of the attributes are equal
5728   if (Fa.hasFnAttribute("sign-return-address") &&
5729       Fb.hasFnAttribute("sign-return-address")) {
5730     StringRef ScopeA =
5731         Fa.getFnAttribute("sign-return-address").getValueAsString();
5732     StringRef ScopeB =
5733         Fb.getFnAttribute("sign-return-address").getValueAsString();
5734     return ScopeA.equals(ScopeB);
5735   }
5736 
5737   // If function B doesn't have the "sign-return-address" attribute but A does,
5738   // the functions' signing behaviour is equal if A's value for
5739   // "sign-return-address" is "none" and vice versa.
5740   if (Fa.hasFnAttribute("sign-return-address")) {
5741     StringRef ScopeA =
5742         Fa.getFnAttribute("sign-return-address").getValueAsString();
5743     return ScopeA.equals("none");
5744   }
5745 
5746   if (Fb.hasFnAttribute("sign-return-address")) {
5747     StringRef ScopeB =
5748         Fb.getFnAttribute("sign-return-address").getValueAsString();
5749     return ScopeB.equals("none");
5750   }
5751 
5752   llvm_unreachable("Unkown combination of sign-return-address attributes");
5753 }
5754 
5755 static bool
5756 outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
5757                                        const outliner::Candidate &b) {
5758   const Function &Fa = a.getMF()->getFunction();
5759   const Function &Fb = b.getMF()->getFunction();
5760 
5761   // If none of the functions have the "sign-return-address-key" attribute
5762   // their keys are equal
5763   if (!Fa.hasFnAttribute("sign-return-address-key") &&
5764       !Fb.hasFnAttribute("sign-return-address-key")) {
5765     return true;
5766   }
5767 
5768   // If both functions have the "sign-return-address-key" attribute their
5769   // keys are equal if the values of "sign-return-address-key" are equal
5770   if (Fa.hasFnAttribute("sign-return-address-key") &&
5771       Fb.hasFnAttribute("sign-return-address-key")) {
5772     StringRef KeyA =
5773         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5774     StringRef KeyB =
5775         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5776     return KeyA.equals(KeyB);
5777   }
5778 
5779   // If B doesn't have the "sign-return-address-key" attribute, both keys are
5780   // equal, if function a has the default key (a_key)
5781   if (Fa.hasFnAttribute("sign-return-address-key")) {
5782     StringRef KeyA =
5783         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5784     return KeyA.equals_lower("a_key");
5785   }
5786 
5787   if (Fb.hasFnAttribute("sign-return-address-key")) {
5788     StringRef KeyB =
5789         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5790     return KeyB.equals_lower("a_key");
5791   }
5792 
5793   llvm_unreachable("Unkown combination of sign-return-address-key attributes");
5794 }
5795 
5796 static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
5797                                                 const outliner::Candidate &b) {
5798   const AArch64Subtarget &SubtargetA =
5799       a.getMF()->getSubtarget<AArch64Subtarget>();
5800   const AArch64Subtarget &SubtargetB =
5801       b.getMF()->getSubtarget<AArch64Subtarget>();
5802   return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
5803 }
5804 
5805 outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
5806     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5807   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5808   unsigned SequenceSize =
5809       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5810                       [this](unsigned Sum, const MachineInstr &MI) {
5811                         return Sum + getInstSizeInBytes(MI);
5812                       });
5813   unsigned NumBytesToCreateFrame = 0;
5814 
5815   // We only allow outlining for functions having exactly matching return
5816   // address signing attributes, i.e., all share the same value for the
5817   // attribute "sign-return-address" and all share the same type of key they
5818   // are signed with.
5819   // Additionally we require all functions to simultaniously either support
5820   // v8.3a features or not. Otherwise an outlined function could get signed
5821   // using dedicated v8.3 instructions and a call from a function that doesn't
5822   // support v8.3 instructions would therefore be invalid.
5823   if (std::adjacent_find(
5824           RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5825           [](const outliner::Candidate &a, const outliner::Candidate &b) {
5826             // Return true if a and b are non-equal w.r.t. return address
5827             // signing or support of v8.3a features
5828             if (outliningCandidatesSigningScopeConsensus(a, b) &&
5829                 outliningCandidatesSigningKeyConsensus(a, b) &&
5830                 outliningCandidatesV8_3OpsConsensus(a, b)) {
5831               return false;
5832             }
5833             return true;
5834           }) != RepeatedSequenceLocs.end()) {
5835     return outliner::OutlinedFunction();
5836   }
5837 
5838   // Since at this point all candidates agree on their return address signing
5839   // picking just one is fine. If the candidate functions potentially sign their
5840   // return addresses, the outlined function should do the same. Note that in
5841   // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5842   // not certainly true that the outlined function will have to sign its return
5843   // address but this decision is made later, when the decision to outline
5844   // has already been made.
5845   // The same holds for the number of additional instructions we need: On
5846   // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5847   // necessary. However, at this point we don't know if the outlined function
5848   // will have a RET instruction so we assume the worst.
5849   const Function &FCF = FirstCand.getMF()->getFunction();
5850   const TargetRegisterInfo &TRI = getRegisterInfo();
5851   if (FCF.hasFnAttribute("sign-return-address")) {
5852     // One PAC and one AUT instructions
5853     NumBytesToCreateFrame += 8;
5854 
5855     // We have to check if sp modifying instructions would get outlined.
5856     // If so we only allow outlining if sp is unchanged overall, so matching
5857     // sub and add instructions are okay to outline, all other sp modifications
5858     // are not
5859     auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5860       int SPValue = 0;
5861       MachineBasicBlock::iterator MBBI = C.front();
5862       for (;;) {
5863         if (MBBI->modifiesRegister(AArch64::SP, &TRI)) {
5864           switch (MBBI->getOpcode()) {
5865           case AArch64::ADDXri:
5866           case AArch64::ADDWri:
5867             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5868             assert(MBBI->getOperand(2).isImm() &&
5869                    "Expected operand to be immediate");
5870             assert(MBBI->getOperand(1).isReg() &&
5871                    "Expected operand to be a register");
5872             // Check if the add just increments sp. If so, we search for
5873             // matching sub instructions that decrement sp. If not, the
5874             // modification is illegal
5875             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5876               SPValue += MBBI->getOperand(2).getImm();
5877             else
5878               return true;
5879             break;
5880           case AArch64::SUBXri:
5881           case AArch64::SUBWri:
5882             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5883             assert(MBBI->getOperand(2).isImm() &&
5884                    "Expected operand to be immediate");
5885             assert(MBBI->getOperand(1).isReg() &&
5886                    "Expected operand to be a register");
5887             // Check if the sub just decrements sp. If so, we search for
5888             // matching add instructions that increment sp. If not, the
5889             // modification is illegal
5890             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5891               SPValue -= MBBI->getOperand(2).getImm();
5892             else
5893               return true;
5894             break;
5895           default:
5896             return true;
5897           }
5898         }
5899         if (MBBI == C.back())
5900           break;
5901         ++MBBI;
5902       }
5903       if (SPValue)
5904         return true;
5905       return false;
5906     };
5907     // Remove candidates with illegal stack modifying instructions
5908     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5909                                               RepeatedSequenceLocs.end(),
5910                                               hasIllegalSPModification),
5911                                RepeatedSequenceLocs.end());
5912 
5913     // If the sequence doesn't have enough candidates left, then we're done.
5914     if (RepeatedSequenceLocs.size() < 2)
5915       return outliner::OutlinedFunction();
5916   }
5917 
5918   // Properties about candidate MBBs that hold for all of them.
5919   unsigned FlagsSetInAll = 0xF;
5920 
5921   // Compute liveness information for each candidate, and set FlagsSetInAll.
5922   std::for_each(RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5923                 [&FlagsSetInAll](outliner::Candidate &C) {
5924                   FlagsSetInAll &= C.Flags;
5925                 });
5926 
5927   // According to the AArch64 Procedure Call Standard, the following are
5928   // undefined on entry/exit from a function call:
5929   //
5930   // * Registers x16, x17, (and thus w16, w17)
5931   // * Condition codes (and thus the NZCV register)
5932   //
5933   // Because if this, we can't outline any sequence of instructions where
5934   // one
5935   // of these registers is live into/across it. Thus, we need to delete
5936   // those
5937   // candidates.
5938   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5939     // If the unsafe registers in this block are all dead, then we don't need
5940     // to compute liveness here.
5941     if (C.Flags & UnsafeRegsDead)
5942       return false;
5943     C.initLRU(TRI);
5944     LiveRegUnits LRU = C.LRU;
5945     return (!LRU.available(AArch64::W16) || !LRU.available(AArch64::W17) ||
5946             !LRU.available(AArch64::NZCV));
5947   };
5948 
5949   // Are there any candidates where those registers are live?
5950   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5951     // Erase every candidate that violates the restrictions above. (It could be
5952     // true that we have viable candidates, so it's not worth bailing out in
5953     // the case that, say, 1 out of 20 candidates violate the restructions.)
5954     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5955                                               RepeatedSequenceLocs.end(),
5956                                               CantGuaranteeValueAcrossCall),
5957                                RepeatedSequenceLocs.end());
5958 
5959     // If the sequence doesn't have enough candidates left, then we're done.
5960     if (RepeatedSequenceLocs.size() < 2)
5961       return outliner::OutlinedFunction();
5962   }
5963 
5964   // At this point, we have only "safe" candidates to outline. Figure out
5965   // frame + call instruction information.
5966 
5967   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5968 
5969   // Helper lambda which sets call information for every candidate.
5970   auto SetCandidateCallInfo =
5971       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5972         for (outliner::Candidate &C : RepeatedSequenceLocs)
5973           C.setCallInfo(CallID, NumBytesForCall);
5974       };
5975 
5976   unsigned FrameID = MachineOutlinerDefault;
5977   NumBytesToCreateFrame += 4;
5978 
5979   bool HasBTI = any_of(RepeatedSequenceLocs, [](outliner::Candidate &C) {
5980     return C.getMF()->getFunction().hasFnAttribute("branch-target-enforcement");
5981   });
5982 
5983   // We check to see if CFI Instructions are present, and if they are
5984   // we find the number of CFI Instructions in the candidates.
5985   unsigned CFICount = 0;
5986   MachineBasicBlock::iterator MBBI = RepeatedSequenceLocs[0].front();
5987   for (unsigned Loc = RepeatedSequenceLocs[0].getStartIdx();
5988        Loc < RepeatedSequenceLocs[0].getEndIdx() + 1; Loc++) {
5989     const std::vector<MCCFIInstruction> &CFIInstructions =
5990         RepeatedSequenceLocs[0].getMF()->getFrameInstructions();
5991     if (MBBI->isCFIInstruction()) {
5992       unsigned CFIIndex = MBBI->getOperand(0).getCFIIndex();
5993       MCCFIInstruction CFI = CFIInstructions[CFIIndex];
5994       CFICount++;
5995     }
5996     MBBI++;
5997   }
5998 
5999   // We compare the number of found CFI Instructions to  the number of CFI
6000   // instructions in the parent function for each candidate.  We must check this
6001   // since if we outline one of the CFI instructions in a function, we have to
6002   // outline them all for correctness. If we do not, the address offsets will be
6003   // incorrect between the two sections of the program.
6004   for (outliner::Candidate &C : RepeatedSequenceLocs) {
6005     std::vector<MCCFIInstruction> CFIInstructions =
6006         C.getMF()->getFrameInstructions();
6007 
6008     if (CFICount > 0 && CFICount != CFIInstructions.size())
6009       return outliner::OutlinedFunction();
6010   }
6011 
6012   // Returns true if an instructions is safe to fix up, false otherwise.
6013   auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
6014     if (MI.isCall())
6015       return true;
6016 
6017     if (!MI.modifiesRegister(AArch64::SP, &TRI) &&
6018         !MI.readsRegister(AArch64::SP, &TRI))
6019       return true;
6020 
6021     // Any modification of SP will break our code to save/restore LR.
6022     // FIXME: We could handle some instructions which add a constant
6023     // offset to SP, with a bit more work.
6024     if (MI.modifiesRegister(AArch64::SP, &TRI))
6025       return false;
6026 
6027     // At this point, we have a stack instruction that we might need to
6028     // fix up. We'll handle it if it's a load or store.
6029     if (MI.mayLoadOrStore()) {
6030       const MachineOperand *Base; // Filled with the base operand of MI.
6031       int64_t Offset;             // Filled with the offset of MI.
6032       bool OffsetIsScalable;
6033 
6034       // Does it allow us to offset the base operand and is the base the
6035       // register SP?
6036       if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable, &TRI) ||
6037           !Base->isReg() || Base->getReg() != AArch64::SP)
6038         return false;
6039 
6040       // Fixe-up code below assumes bytes.
6041       if (OffsetIsScalable)
6042         return false;
6043 
6044       // Find the minimum/maximum offset for this instruction and check
6045       // if fixing it up would be in range.
6046       int64_t MinOffset,
6047           MaxOffset;  // Unscaled offsets for the instruction.
6048       TypeSize Scale(0U, false); // The scale to multiply the offsets by.
6049       unsigned DummyWidth;
6050       getMemOpInfo(MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
6051 
6052       Offset += 16; // Update the offset to what it would be if we outlined.
6053       if (Offset < MinOffset * (int64_t)Scale.getFixedSize() ||
6054           Offset > MaxOffset * (int64_t)Scale.getFixedSize())
6055         return false;
6056 
6057       // It's in range, so we can outline it.
6058       return true;
6059     }
6060 
6061     // FIXME: Add handling for instructions like "add x0, sp, #8".
6062 
6063     // We can't fix it up, so don't outline it.
6064     return false;
6065   };
6066 
6067   // True if it's possible to fix up each stack instruction in this sequence.
6068   // Important for frames/call variants that modify the stack.
6069   bool AllStackInstrsSafe = std::all_of(
6070       FirstCand.front(), std::next(FirstCand.back()), IsSafeToFixup);
6071 
6072   // If the last instruction in any candidate is a terminator, then we should
6073   // tail call all of the candidates.
6074   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
6075     FrameID = MachineOutlinerTailCall;
6076     NumBytesToCreateFrame = 0;
6077     SetCandidateCallInfo(MachineOutlinerTailCall, 4);
6078   }
6079 
6080   else if (LastInstrOpcode == AArch64::BL ||
6081            (LastInstrOpcode == AArch64::BLR && !HasBTI)) {
6082     // FIXME: Do we need to check if the code after this uses the value of LR?
6083     FrameID = MachineOutlinerThunk;
6084     NumBytesToCreateFrame = 0;
6085     SetCandidateCallInfo(MachineOutlinerThunk, 4);
6086   }
6087 
6088   else {
6089     // We need to decide how to emit calls + frames. We can always emit the same
6090     // frame if we don't need to save to the stack. If we have to save to the
6091     // stack, then we need a different frame.
6092     unsigned NumBytesNoStackCalls = 0;
6093     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
6094 
6095     // Check if we have to save LR.
6096     for (outliner::Candidate &C : RepeatedSequenceLocs) {
6097       C.initLRU(TRI);
6098 
6099       // If we have a noreturn caller, then we're going to be conservative and
6100       // say that we have to save LR. If we don't have a ret at the end of the
6101       // block, then we can't reason about liveness accurately.
6102       //
6103       // FIXME: We can probably do better than always disabling this in
6104       // noreturn functions by fixing up the liveness info.
6105       bool IsNoReturn =
6106           C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
6107 
6108       // Is LR available? If so, we don't need a save.
6109       if (C.LRU.available(AArch64::LR) && !IsNoReturn) {
6110         NumBytesNoStackCalls += 4;
6111         C.setCallInfo(MachineOutlinerNoLRSave, 4);
6112         CandidatesWithoutStackFixups.push_back(C);
6113       }
6114 
6115       // Is an unused register available? If so, we won't modify the stack, so
6116       // we can outline with the same frame type as those that don't save LR.
6117       else if (findRegisterToSaveLRTo(C)) {
6118         NumBytesNoStackCalls += 12;
6119         C.setCallInfo(MachineOutlinerRegSave, 12);
6120         CandidatesWithoutStackFixups.push_back(C);
6121       }
6122 
6123       // Is SP used in the sequence at all? If not, we don't have to modify
6124       // the stack, so we are guaranteed to get the same frame.
6125       else if (C.UsedInSequence.available(AArch64::SP)) {
6126         NumBytesNoStackCalls += 12;
6127         C.setCallInfo(MachineOutlinerDefault, 12);
6128         CandidatesWithoutStackFixups.push_back(C);
6129       }
6130 
6131       // If we outline this, we need to modify the stack. Pretend we don't
6132       // outline this by saving all of its bytes.
6133       else {
6134         NumBytesNoStackCalls += SequenceSize;
6135       }
6136     }
6137 
6138     // If there are no places where we have to save LR, then note that we
6139     // don't have to update the stack. Otherwise, give every candidate the
6140     // default call type, as long as it's safe to do so.
6141     if (!AllStackInstrsSafe ||
6142         NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
6143       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
6144       FrameID = MachineOutlinerNoLRSave;
6145     } else {
6146       SetCandidateCallInfo(MachineOutlinerDefault, 12);
6147     }
6148 
6149     // If we dropped all of the candidates, bail out here.
6150     if (RepeatedSequenceLocs.size() < 2) {
6151       RepeatedSequenceLocs.clear();
6152       return outliner::OutlinedFunction();
6153     }
6154   }
6155 
6156   // Does every candidate's MBB contain a call? If so, then we might have a call
6157   // in the range.
6158   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6159     // Check if the range contains a call. These require a save + restore of the
6160     // link register.
6161     bool ModStackToSaveLR = false;
6162     if (std::any_of(FirstCand.front(), FirstCand.back(),
6163                     [](const MachineInstr &MI) { return MI.isCall(); }))
6164       ModStackToSaveLR = true;
6165 
6166     // Handle the last instruction separately. If this is a tail call, then the
6167     // last instruction is a call. We don't want to save + restore in this case.
6168     // However, it could be possible that the last instruction is a call without
6169     // it being valid to tail call this sequence. We should consider this as
6170     // well.
6171     else if (FrameID != MachineOutlinerThunk &&
6172              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6173       ModStackToSaveLR = true;
6174 
6175     if (ModStackToSaveLR) {
6176       // We can't fix up the stack. Bail out.
6177       if (!AllStackInstrsSafe) {
6178         RepeatedSequenceLocs.clear();
6179         return outliner::OutlinedFunction();
6180       }
6181 
6182       // Save + restore LR.
6183       NumBytesToCreateFrame += 8;
6184     }
6185   }
6186 
6187   // If we have CFI instructions, we can only outline if the outlined section
6188   // can be a tail call
6189   if (FrameID != MachineOutlinerTailCall && CFICount > 0)
6190     return outliner::OutlinedFunction();
6191 
6192   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6193                                     NumBytesToCreateFrame, FrameID);
6194 }
6195 
6196 bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
6197     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6198   const Function &F = MF.getFunction();
6199 
6200   // Can F be deduplicated by the linker? If it can, don't outline from it.
6201   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6202     return false;
6203 
6204   // Don't outline from functions with section markings; the program could
6205   // expect that all the code is in the named section.
6206   // FIXME: Allow outlining from multiple functions with the same section
6207   // marking.
6208   if (F.hasSection())
6209     return false;
6210 
6211   // Outlining from functions with redzones is unsafe since the outliner may
6212   // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
6213   // outline from it.
6214   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
6215   if (!AFI || AFI->hasRedZone().getValueOr(true))
6216     return false;
6217 
6218   // FIXME: Teach the outliner to generate/handle Windows unwind info.
6219   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
6220     return false;
6221 
6222   // It's safe to outline from MF.
6223   return true;
6224 }
6225 
6226 bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6227                                               unsigned &Flags) const {
6228   // Check if LR is available through all of the MBB. If it's not, then set
6229   // a flag.
6230   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6231          "Suitable Machine Function for outlining must track liveness");
6232   LiveRegUnits LRU(getRegisterInfo());
6233 
6234   std::for_each(MBB.rbegin(), MBB.rend(),
6235                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6236 
6237   // Check if each of the unsafe registers are available...
6238   bool W16AvailableInBlock = LRU.available(AArch64::W16);
6239   bool W17AvailableInBlock = LRU.available(AArch64::W17);
6240   bool NZCVAvailableInBlock = LRU.available(AArch64::NZCV);
6241 
6242   // If all of these are dead (and not live out), we know we don't have to check
6243   // them later.
6244   if (W16AvailableInBlock && W17AvailableInBlock && NZCVAvailableInBlock)
6245     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6246 
6247   // Now, add the live outs to the set.
6248   LRU.addLiveOuts(MBB);
6249 
6250   // If any of these registers is available in the MBB, but also a live out of
6251   // the block, then we know outlining is unsafe.
6252   if (W16AvailableInBlock && !LRU.available(AArch64::W16))
6253     return false;
6254   if (W17AvailableInBlock && !LRU.available(AArch64::W17))
6255     return false;
6256   if (NZCVAvailableInBlock && !LRU.available(AArch64::NZCV))
6257     return false;
6258 
6259   // Check if there's a call inside this MachineBasicBlock. If there is, then
6260   // set a flag.
6261   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6262     Flags |= MachineOutlinerMBBFlags::HasCalls;
6263 
6264   MachineFunction *MF = MBB.getParent();
6265 
6266   // In the event that we outline, we may have to save LR. If there is an
6267   // available register in the MBB, then we'll always save LR there. Check if
6268   // this is true.
6269   bool CanSaveLR = false;
6270   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
6271       MF->getSubtarget().getRegisterInfo());
6272 
6273   // Check if there is an available register across the sequence that we can
6274   // use.
6275   for (unsigned Reg : AArch64::GPR64RegClass) {
6276     if (!ARI->isReservedReg(*MF, Reg) && Reg != AArch64::LR &&
6277         Reg != AArch64::X16 && Reg != AArch64::X17 && LRU.available(Reg)) {
6278       CanSaveLR = true;
6279       break;
6280     }
6281   }
6282 
6283   // Check if we have a register we can save LR to, and if LR was used
6284   // somewhere. If both of those things are true, then we need to evaluate the
6285   // safety of outlining stack instructions later.
6286   if (!CanSaveLR && !LRU.available(AArch64::LR))
6287     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6288 
6289   return true;
6290 }
6291 
6292 outliner::InstrType
6293 AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6294                                    unsigned Flags) const {
6295   MachineInstr &MI = *MIT;
6296   MachineBasicBlock *MBB = MI.getParent();
6297   MachineFunction *MF = MBB->getParent();
6298   AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
6299 
6300   // Don't outline anything used for return address signing. The outlined
6301   // function will get signed later if needed
6302   switch (MI.getOpcode()) {
6303   case AArch64::PACIASP:
6304   case AArch64::PACIBSP:
6305   case AArch64::AUTIASP:
6306   case AArch64::AUTIBSP:
6307   case AArch64::RETAA:
6308   case AArch64::RETAB:
6309   case AArch64::EMITBKEY:
6310     return outliner::InstrType::Illegal;
6311   }
6312 
6313   // Don't outline LOHs.
6314   if (FuncInfo->getLOHRelated().count(&MI))
6315     return outliner::InstrType::Illegal;
6316 
6317   // We can only outline these if we will tail call the outlined function, or
6318   // fix up the CFI offsets. Currently, CFI instructions are outlined only if
6319   // in a tail call.
6320   //
6321   // FIXME: If the proper fixups for the offset are implemented, this should be
6322   // possible.
6323   if (MI.isCFIInstruction())
6324     return outliner::InstrType::Legal;
6325 
6326   // Don't allow debug values to impact outlining type.
6327   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6328     return outliner::InstrType::Invisible;
6329 
6330   // At this point, KILL instructions don't really tell us much so we can go
6331   // ahead and skip over them.
6332   if (MI.isKill())
6333     return outliner::InstrType::Invisible;
6334 
6335   // Is this a terminator for a basic block?
6336   if (MI.isTerminator()) {
6337 
6338     // Is this the end of a function?
6339     if (MI.getParent()->succ_empty())
6340       return outliner::InstrType::Legal;
6341 
6342     // It's not, so don't outline it.
6343     return outliner::InstrType::Illegal;
6344   }
6345 
6346   // Make sure none of the operands are un-outlinable.
6347   for (const MachineOperand &MOP : MI.operands()) {
6348     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6349         MOP.isTargetIndex())
6350       return outliner::InstrType::Illegal;
6351 
6352     // If it uses LR or W30 explicitly, then don't touch it.
6353     if (MOP.isReg() && !MOP.isImplicit() &&
6354         (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
6355       return outliner::InstrType::Illegal;
6356   }
6357 
6358   // Special cases for instructions that can always be outlined, but will fail
6359   // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
6360   // be outlined because they don't require a *specific* value to be in LR.
6361   if (MI.getOpcode() == AArch64::ADRP)
6362     return outliner::InstrType::Legal;
6363 
6364   // If MI is a call we might be able to outline it. We don't want to outline
6365   // any calls that rely on the position of items on the stack. When we outline
6366   // something containing a call, we have to emit a save and restore of LR in
6367   // the outlined function. Currently, this always happens by saving LR to the
6368   // stack. Thus, if we outline, say, half the parameters for a function call
6369   // plus the call, then we'll break the callee's expectations for the layout
6370   // of the stack.
6371   //
6372   // FIXME: Allow calls to functions which construct a stack frame, as long
6373   // as they don't access arguments on the stack.
6374   // FIXME: Figure out some way to analyze functions defined in other modules.
6375   // We should be able to compute the memory usage based on the IR calling
6376   // convention, even if we can't see the definition.
6377   if (MI.isCall()) {
6378     // Get the function associated with the call. Look at each operand and find
6379     // the one that represents the callee and get its name.
6380     const Function *Callee = nullptr;
6381     for (const MachineOperand &MOP : MI.operands()) {
6382       if (MOP.isGlobal()) {
6383         Callee = dyn_cast<Function>(MOP.getGlobal());
6384         break;
6385       }
6386     }
6387 
6388     // Never outline calls to mcount.  There isn't any rule that would require
6389     // this, but the Linux kernel's "ftrace" feature depends on it.
6390     if (Callee && Callee->getName() == "\01_mcount")
6391       return outliner::InstrType::Illegal;
6392 
6393     // If we don't know anything about the callee, assume it depends on the
6394     // stack layout of the caller. In that case, it's only legal to outline
6395     // as a tail-call.  Whitelist the call instructions we know about so we
6396     // don't get unexpected results with call pseudo-instructions.
6397     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6398     if (MI.getOpcode() == AArch64::BLR || MI.getOpcode() == AArch64::BL)
6399       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6400 
6401     if (!Callee)
6402       return UnknownCallOutlineType;
6403 
6404     // We have a function we have information about. Check it if it's something
6405     // can safely outline.
6406     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6407 
6408     // We don't know what's going on with the callee at all. Don't touch it.
6409     if (!CalleeMF)
6410       return UnknownCallOutlineType;
6411 
6412     // Check if we know anything about the callee saves on the function. If we
6413     // don't, then don't touch it, since that implies that we haven't
6414     // computed anything about its stack frame yet.
6415     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6416     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6417         MFI.getNumObjects() > 0)
6418       return UnknownCallOutlineType;
6419 
6420     // At this point, we can say that CalleeMF ought to not pass anything on the
6421     // stack. Therefore, we can outline it.
6422     return outliner::InstrType::Legal;
6423   }
6424 
6425   // Don't outline positions.
6426   if (MI.isPosition())
6427     return outliner::InstrType::Illegal;
6428 
6429   // Don't touch the link register or W30.
6430   if (MI.readsRegister(AArch64::W30, &getRegisterInfo()) ||
6431       MI.modifiesRegister(AArch64::W30, &getRegisterInfo()))
6432     return outliner::InstrType::Illegal;
6433 
6434   // Don't outline BTI instructions, because that will prevent the outlining
6435   // site from being indirectly callable.
6436   if (MI.getOpcode() == AArch64::HINT) {
6437     int64_t Imm = MI.getOperand(0).getImm();
6438     if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
6439       return outliner::InstrType::Illegal;
6440   }
6441 
6442   return outliner::InstrType::Legal;
6443 }
6444 
6445 void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6446   for (MachineInstr &MI : MBB) {
6447     const MachineOperand *Base;
6448     unsigned Width;
6449     int64_t Offset;
6450     bool OffsetIsScalable;
6451 
6452     // Is this a load or store with an immediate offset with SP as the base?
6453     if (!MI.mayLoadOrStore() ||
6454         !getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable, Width,
6455                                       &RI) ||
6456         (Base->isReg() && Base->getReg() != AArch64::SP))
6457       continue;
6458 
6459     // It is, so we have to fix it up.
6460     TypeSize Scale(0U, false);
6461     int64_t Dummy1, Dummy2;
6462 
6463     MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(MI);
6464     assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
6465     getMemOpInfo(MI.getOpcode(), Scale, Width, Dummy1, Dummy2);
6466     assert(Scale != 0 && "Unexpected opcode!");
6467     assert(!OffsetIsScalable && "Expected offset to be a byte offset");
6468 
6469     // We've pushed the return address to the stack, so add 16 to the offset.
6470     // This is safe, since we already checked if it would overflow when we
6471     // checked if this instruction was legal to outline.
6472     int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedSize();
6473     StackOffsetOperand.setImm(NewImm);
6474   }
6475 }
6476 
6477 static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
6478                                  bool ShouldSignReturnAddr,
6479                                  bool ShouldSignReturnAddrWithAKey) {
6480   if (ShouldSignReturnAddr) {
6481     MachineBasicBlock::iterator MBBPAC = MBB.begin();
6482     MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator();
6483     const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
6484     const TargetInstrInfo *TII = Subtarget.getInstrInfo();
6485     DebugLoc DL;
6486 
6487     if (MBBAUT != MBB.end())
6488       DL = MBBAUT->getDebugLoc();
6489 
6490     // At the very beginning of the basic block we insert the following
6491     // depending on the key type
6492     //
6493     // a_key:                   b_key:
6494     //    PACIASP                   EMITBKEY
6495     //    CFI_INSTRUCTION           PACIBSP
6496     //                              CFI_INSTRUCTION
6497     if (ShouldSignReturnAddrWithAKey) {
6498       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIASP))
6499           .setMIFlag(MachineInstr::FrameSetup);
6500     } else {
6501       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::EMITBKEY))
6502           .setMIFlag(MachineInstr::FrameSetup);
6503       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIBSP))
6504           .setMIFlag(MachineInstr::FrameSetup);
6505     }
6506     unsigned CFIIndex =
6507         MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
6508     BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::CFI_INSTRUCTION))
6509         .addCFIIndex(CFIIndex)
6510         .setMIFlags(MachineInstr::FrameSetup);
6511 
6512     // If v8.3a features are available we can replace a RET instruction by
6513     // RETAA or RETAB and omit the AUT instructions
6514     if (Subtarget.hasV8_3aOps() && MBBAUT != MBB.end() &&
6515         MBBAUT->getOpcode() == AArch64::RET) {
6516       BuildMI(MBB, MBBAUT, DL,
6517               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6518                                                     : AArch64::RETAB))
6519           .copyImplicitOps(*MBBAUT);
6520       MBB.erase(MBBAUT);
6521     } else {
6522       BuildMI(MBB, MBBAUT, DL,
6523               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6524                                                     : AArch64::AUTIBSP))
6525           .setMIFlag(MachineInstr::FrameDestroy);
6526     }
6527   }
6528 }
6529 
6530 void AArch64InstrInfo::buildOutlinedFrame(
6531     MachineBasicBlock &MBB, MachineFunction &MF,
6532     const outliner::OutlinedFunction &OF) const {
6533 
6534   AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
6535 
6536   if (OF.FrameConstructionID == MachineOutlinerTailCall)
6537     FI->setOutliningStyle("Tail Call");
6538   else if (OF.FrameConstructionID == MachineOutlinerThunk) {
6539     // For thunk outlining, rewrite the last instruction from a call to a
6540     // tail-call.
6541     MachineInstr *Call = &*--MBB.instr_end();
6542     unsigned TailOpcode;
6543     if (Call->getOpcode() == AArch64::BL) {
6544       TailOpcode = AArch64::TCRETURNdi;
6545     } else {
6546       assert(Call->getOpcode() == AArch64::BLR);
6547       TailOpcode = AArch64::TCRETURNriALL;
6548     }
6549     MachineInstr *TC = BuildMI(MF, DebugLoc(), get(TailOpcode))
6550                            .add(Call->getOperand(0))
6551                            .addImm(0);
6552     MBB.insert(MBB.end(), TC);
6553     Call->eraseFromParent();
6554 
6555     FI->setOutliningStyle("Thunk");
6556   }
6557 
6558   bool IsLeafFunction = true;
6559 
6560   // Is there a call in the outlined range?
6561   auto IsNonTailCall = [](const MachineInstr &MI) {
6562     return MI.isCall() && !MI.isReturn();
6563   };
6564 
6565   if (std::any_of(MBB.instr_begin(), MBB.instr_end(), IsNonTailCall)) {
6566     // Fix up the instructions in the range, since we're going to modify the
6567     // stack.
6568     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6569            "Can only fix up stack references once");
6570     fixupPostOutline(MBB);
6571 
6572     IsLeafFunction = false;
6573 
6574     // LR has to be a live in so that we can save it.
6575     if (!MBB.isLiveIn(AArch64::LR))
6576       MBB.addLiveIn(AArch64::LR);
6577 
6578     MachineBasicBlock::iterator It = MBB.begin();
6579     MachineBasicBlock::iterator Et = MBB.end();
6580 
6581     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6582         OF.FrameConstructionID == MachineOutlinerThunk)
6583       Et = std::prev(MBB.end());
6584 
6585     // Insert a save before the outlined region
6586     MachineInstr *STRXpre = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6587                                 .addReg(AArch64::SP, RegState::Define)
6588                                 .addReg(AArch64::LR)
6589                                 .addReg(AArch64::SP)
6590                                 .addImm(-16);
6591     It = MBB.insert(It, STRXpre);
6592 
6593     const TargetSubtargetInfo &STI = MF.getSubtarget();
6594     const MCRegisterInfo *MRI = STI.getRegisterInfo();
6595     unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true);
6596 
6597     // Add a CFI saying the stack was moved 16 B down.
6598     int64_t StackPosEntry =
6599         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 16));
6600     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6601         .addCFIIndex(StackPosEntry)
6602         .setMIFlags(MachineInstr::FrameSetup);
6603 
6604     // Add a CFI saying that the LR that we want to find is now 16 B higher than
6605     // before.
6606     int64_t LRPosEntry =
6607         MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfReg, -16));
6608     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6609         .addCFIIndex(LRPosEntry)
6610         .setMIFlags(MachineInstr::FrameSetup);
6611 
6612     // Insert a restore before the terminator for the function.
6613     MachineInstr *LDRXpost = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6614                                  .addReg(AArch64::SP, RegState::Define)
6615                                  .addReg(AArch64::LR, RegState::Define)
6616                                  .addReg(AArch64::SP)
6617                                  .addImm(16);
6618     Et = MBB.insert(Et, LDRXpost);
6619   }
6620 
6621   // If a bunch of candidates reach this point they must agree on their return
6622   // address signing. It is therefore enough to just consider the signing
6623   // behaviour of one of them
6624   const Function &CF = OF.Candidates.front().getMF()->getFunction();
6625   bool ShouldSignReturnAddr = false;
6626   if (CF.hasFnAttribute("sign-return-address")) {
6627     StringRef Scope =
6628         CF.getFnAttribute("sign-return-address").getValueAsString();
6629     if (Scope.equals("all"))
6630       ShouldSignReturnAddr = true;
6631     else if (Scope.equals("non-leaf") && !IsLeafFunction)
6632       ShouldSignReturnAddr = true;
6633   }
6634 
6635   // a_key is the default
6636   bool ShouldSignReturnAddrWithAKey = true;
6637   if (CF.hasFnAttribute("sign-return-address-key")) {
6638     const StringRef Key =
6639         CF.getFnAttribute("sign-return-address-key").getValueAsString();
6640     // Key can either be a_key or b_key
6641     assert((Key.equals_lower("a_key") || Key.equals_lower("b_key")) &&
6642            "Return address signing key must be either a_key or b_key");
6643     ShouldSignReturnAddrWithAKey = Key.equals_lower("a_key");
6644   }
6645 
6646   // If this is a tail call outlined function, then there's already a return.
6647   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6648       OF.FrameConstructionID == MachineOutlinerThunk) {
6649     signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6650                          ShouldSignReturnAddrWithAKey);
6651     return;
6652   }
6653 
6654   // It's not a tail call, so we have to insert the return ourselves.
6655 
6656   // LR has to be a live in so that we can return to it.
6657   if (!MBB.isLiveIn(AArch64::LR))
6658     MBB.addLiveIn(AArch64::LR);
6659 
6660   MachineInstr *ret = BuildMI(MF, DebugLoc(), get(AArch64::RET))
6661                           .addReg(AArch64::LR);
6662   MBB.insert(MBB.end(), ret);
6663 
6664   signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6665                        ShouldSignReturnAddrWithAKey);
6666 
6667   FI->setOutliningStyle("Function");
6668 
6669   // Did we have to modify the stack by saving the link register?
6670   if (OF.FrameConstructionID != MachineOutlinerDefault)
6671     return;
6672 
6673   // We modified the stack.
6674   // Walk over the basic block and fix up all the stack accesses.
6675   fixupPostOutline(MBB);
6676 }
6677 
6678 MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
6679     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6680     MachineFunction &MF, const outliner::Candidate &C) const {
6681 
6682   // Are we tail calling?
6683   if (C.CallConstructionID == MachineOutlinerTailCall) {
6684     // If yes, then we can just branch to the label.
6685     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::TCRETURNdi))
6686                             .addGlobalAddress(M.getNamedValue(MF.getName()))
6687                             .addImm(0));
6688     return It;
6689   }
6690 
6691   // Are we saving the link register?
6692   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6693       C.CallConstructionID == MachineOutlinerThunk) {
6694     // No, so just insert the call.
6695     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6696                             .addGlobalAddress(M.getNamedValue(MF.getName())));
6697     return It;
6698   }
6699 
6700   // We want to return the spot where we inserted the call.
6701   MachineBasicBlock::iterator CallPt;
6702 
6703   // Instructions for saving and restoring LR around the call instruction we're
6704   // going to insert.
6705   MachineInstr *Save;
6706   MachineInstr *Restore;
6707   // Can we save to a register?
6708   if (C.CallConstructionID == MachineOutlinerRegSave) {
6709     // FIXME: This logic should be sunk into a target-specific interface so that
6710     // we don't have to recompute the register.
6711     unsigned Reg = findRegisterToSaveLRTo(C);
6712     assert(Reg != 0 && "No callee-saved register available?");
6713 
6714     // Save and restore LR from that register.
6715     Save = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), Reg)
6716                .addReg(AArch64::XZR)
6717                .addReg(AArch64::LR)
6718                .addImm(0);
6719     Restore = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
6720                 .addReg(AArch64::XZR)
6721                 .addReg(Reg)
6722                 .addImm(0);
6723   } else {
6724     // We have the default case. Save and restore from SP.
6725     Save = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6726                .addReg(AArch64::SP, RegState::Define)
6727                .addReg(AArch64::LR)
6728                .addReg(AArch64::SP)
6729                .addImm(-16);
6730     Restore = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6731                   .addReg(AArch64::SP, RegState::Define)
6732                   .addReg(AArch64::LR, RegState::Define)
6733                   .addReg(AArch64::SP)
6734                   .addImm(16);
6735   }
6736 
6737   It = MBB.insert(It, Save);
6738   It++;
6739 
6740   // Insert the call.
6741   It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6742                           .addGlobalAddress(M.getNamedValue(MF.getName())));
6743   CallPt = It;
6744   It++;
6745 
6746   It = MBB.insert(It, Restore);
6747   return CallPt;
6748 }
6749 
6750 bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
6751   MachineFunction &MF) const {
6752   return MF.getFunction().hasMinSize();
6753 }
6754 
6755 Optional<DestSourcePair>
6756 AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
6757 
6758   // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
6759   // and zero immediate operands used as an alias for mov instruction.
6760   if (MI.getOpcode() == AArch64::ORRWrs &&
6761       MI.getOperand(1).getReg() == AArch64::WZR &&
6762       MI.getOperand(3).getImm() == 0x0) {
6763     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6764   }
6765 
6766   if (MI.getOpcode() == AArch64::ORRXrs &&
6767       MI.getOperand(1).getReg() == AArch64::XZR &&
6768       MI.getOperand(3).getImm() == 0x0) {
6769     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6770   }
6771 
6772   return None;
6773 }
6774 
6775 Optional<RegImmPair> AArch64InstrInfo::isAddImmediate(const MachineInstr &MI,
6776                                                       Register Reg) const {
6777   int Sign = 1;
6778   int64_t Offset = 0;
6779 
6780   // TODO: Handle cases where Reg is a super- or sub-register of the
6781   // destination register.
6782   const MachineOperand &Op0 = MI.getOperand(0);
6783   if (!Op0.isReg() || Reg != Op0.getReg())
6784     return None;
6785 
6786   switch (MI.getOpcode()) {
6787   default:
6788     return None;
6789   case AArch64::SUBWri:
6790   case AArch64::SUBXri:
6791   case AArch64::SUBSWri:
6792   case AArch64::SUBSXri:
6793     Sign *= -1;
6794     LLVM_FALLTHROUGH;
6795   case AArch64::ADDSWri:
6796   case AArch64::ADDSXri:
6797   case AArch64::ADDWri:
6798   case AArch64::ADDXri: {
6799     // TODO: Third operand can be global address (usually some string).
6800     if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg() ||
6801         !MI.getOperand(2).isImm())
6802       return None;
6803     Offset = MI.getOperand(2).getImm() * Sign;
6804     int Shift = MI.getOperand(3).getImm();
6805     assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
6806     Offset = Offset << Shift;
6807   }
6808   }
6809   return RegImmPair{MI.getOperand(1).getReg(), Offset};
6810 }
6811 
6812 /// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
6813 /// the destination register then, if possible, describe the value in terms of
6814 /// the source register.
6815 static Optional<ParamLoadedValue>
6816 describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
6817                        const TargetInstrInfo *TII,
6818                        const TargetRegisterInfo *TRI) {
6819   auto DestSrc = TII->isCopyInstr(MI);
6820   if (!DestSrc)
6821     return None;
6822 
6823   Register DestReg = DestSrc->Destination->getReg();
6824   Register SrcReg = DestSrc->Source->getReg();
6825 
6826   auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
6827 
6828   // If the described register is the destination, just return the source.
6829   if (DestReg == DescribedReg)
6830     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6831 
6832   // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
6833   if (MI.getOpcode() == AArch64::ORRWrs &&
6834       TRI->isSuperRegister(DestReg, DescribedReg))
6835     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6836 
6837   // We may need to describe the lower part of a ORRXrs move.
6838   if (MI.getOpcode() == AArch64::ORRXrs &&
6839       TRI->isSubRegister(DestReg, DescribedReg)) {
6840     Register SrcSubReg = TRI->getSubReg(SrcReg, AArch64::sub_32);
6841     return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr);
6842   }
6843 
6844   assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
6845          "Unhandled ORR[XW]rs copy case");
6846 
6847   return None;
6848 }
6849 
6850 Optional<ParamLoadedValue>
6851 AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
6852                                       Register Reg) const {
6853   const MachineFunction *MF = MI.getMF();
6854   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6855   switch (MI.getOpcode()) {
6856   case AArch64::MOVZWi:
6857   case AArch64::MOVZXi: {
6858     // MOVZWi may be used for producing zero-extended 32-bit immediates in
6859     // 64-bit parameters, so we need to consider super-registers.
6860     if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
6861       return None;
6862 
6863     if (!MI.getOperand(1).isImm())
6864       return None;
6865     int64_t Immediate = MI.getOperand(1).getImm();
6866     int Shift = MI.getOperand(2).getImm();
6867     return ParamLoadedValue(MachineOperand::CreateImm(Immediate << Shift),
6868                             nullptr);
6869   }
6870   case AArch64::ORRWrs:
6871   case AArch64::ORRXrs:
6872     return describeORRLoadedValue(MI, Reg, this, TRI);
6873   }
6874 
6875   return TargetInstrInfo::describeLoadedValue(MI, Reg);
6876 }
6877 
6878 uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
6879   return get(Opc).TSFlags & AArch64::ElementSizeMask;
6880 }
6881 
6882 #define GET_INSTRINFO_HELPERS
6883 #define GET_INSTRMAP_INFO
6884 #include "AArch64GenInstrInfo.inc"
6885