1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64InstrInfo.h"
14 #include "AArch64MachineFunctionInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "Utils/AArch64BaseInfo.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineOperand.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/MachineModuleInfo.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetRegisterInfo.h"
32 #include "llvm/CodeGen/TargetSubtargetInfo.h"
33 #include "llvm/IR/DebugInfoMetadata.h"
34 #include "llvm/IR/DebugLoc.h"
35 #include "llvm/IR/GlobalValue.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/MCInstrDesc.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Compiler.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <cassert>
48 #include <cstdint>
49 #include <iterator>
50 #include <utility>
51 
52 using namespace llvm;
53 
54 #define GET_INSTRINFO_CTOR_DTOR
55 #include "AArch64GenInstrInfo.inc"
56 
57 static cl::opt<unsigned> TBZDisplacementBits(
58     "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
59     cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
60 
61 static cl::opt<unsigned> CBZDisplacementBits(
62     "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
63     cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
64 
65 static cl::opt<unsigned>
66     BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
67                         cl::desc("Restrict range of Bcc instructions (DEBUG)"));
68 
69 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
70     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
71                           AArch64::CATCHRET),
72       RI(STI.getTargetTriple()), Subtarget(STI) {}
73 
74 /// GetInstSize - Return the number of bytes of code the specified
75 /// instruction may be.  This returns the maximum number of bytes.
76 unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
77   const MachineBasicBlock &MBB = *MI.getParent();
78   const MachineFunction *MF = MBB.getParent();
79   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
80 
81   {
82     auto Op = MI.getOpcode();
83     if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
84       return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
85   }
86 
87   // Meta-instructions emit no code.
88   if (MI.isMetaInstruction())
89     return 0;
90 
91   // FIXME: We currently only handle pseudoinstructions that don't get expanded
92   //        before the assembly printer.
93   unsigned NumBytes = 0;
94   const MCInstrDesc &Desc = MI.getDesc();
95   switch (Desc.getOpcode()) {
96   default:
97     // Anything not explicitly designated otherwise is a normal 4-byte insn.
98     NumBytes = 4;
99     break;
100   case TargetOpcode::STACKMAP:
101     // The upper bound for a stackmap intrinsic is the full length of its shadow
102     NumBytes = StackMapOpers(&MI).getNumPatchBytes();
103     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
104     break;
105   case TargetOpcode::PATCHPOINT:
106     // The size of the patchpoint intrinsic is the number of bytes requested
107     NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
108     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
109     break;
110   case AArch64::TLSDESC_CALLSEQ:
111     // This gets lowered to an instruction sequence which takes 16 bytes
112     NumBytes = 16;
113     break;
114   case AArch64::JumpTableDest32:
115   case AArch64::JumpTableDest16:
116   case AArch64::JumpTableDest8:
117     NumBytes = 12;
118     break;
119   case AArch64::SPACE:
120     NumBytes = MI.getOperand(1).getImm();
121     break;
122   }
123 
124   return NumBytes;
125 }
126 
127 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
128                             SmallVectorImpl<MachineOperand> &Cond) {
129   // Block ends with fall-through condbranch.
130   switch (LastInst->getOpcode()) {
131   default:
132     llvm_unreachable("Unknown branch instruction?");
133   case AArch64::Bcc:
134     Target = LastInst->getOperand(1).getMBB();
135     Cond.push_back(LastInst->getOperand(0));
136     break;
137   case AArch64::CBZW:
138   case AArch64::CBZX:
139   case AArch64::CBNZW:
140   case AArch64::CBNZX:
141     Target = LastInst->getOperand(1).getMBB();
142     Cond.push_back(MachineOperand::CreateImm(-1));
143     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
144     Cond.push_back(LastInst->getOperand(0));
145     break;
146   case AArch64::TBZW:
147   case AArch64::TBZX:
148   case AArch64::TBNZW:
149   case AArch64::TBNZX:
150     Target = LastInst->getOperand(2).getMBB();
151     Cond.push_back(MachineOperand::CreateImm(-1));
152     Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
153     Cond.push_back(LastInst->getOperand(0));
154     Cond.push_back(LastInst->getOperand(1));
155   }
156 }
157 
158 static unsigned getBranchDisplacementBits(unsigned Opc) {
159   switch (Opc) {
160   default:
161     llvm_unreachable("unexpected opcode!");
162   case AArch64::B:
163     return 64;
164   case AArch64::TBNZW:
165   case AArch64::TBZW:
166   case AArch64::TBNZX:
167   case AArch64::TBZX:
168     return TBZDisplacementBits;
169   case AArch64::CBNZW:
170   case AArch64::CBZW:
171   case AArch64::CBNZX:
172   case AArch64::CBZX:
173     return CBZDisplacementBits;
174   case AArch64::Bcc:
175     return BCCDisplacementBits;
176   }
177 }
178 
179 bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
180                                              int64_t BrOffset) const {
181   unsigned Bits = getBranchDisplacementBits(BranchOp);
182   assert(Bits >= 3 && "max branch displacement must be enough to jump"
183                       "over conditional branch expansion");
184   return isIntN(Bits, BrOffset / 4);
185 }
186 
187 MachineBasicBlock *
188 AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
189   switch (MI.getOpcode()) {
190   default:
191     llvm_unreachable("unexpected opcode!");
192   case AArch64::B:
193     return MI.getOperand(0).getMBB();
194   case AArch64::TBZW:
195   case AArch64::TBNZW:
196   case AArch64::TBZX:
197   case AArch64::TBNZX:
198     return MI.getOperand(2).getMBB();
199   case AArch64::CBZW:
200   case AArch64::CBNZW:
201   case AArch64::CBZX:
202   case AArch64::CBNZX:
203   case AArch64::Bcc:
204     return MI.getOperand(1).getMBB();
205   }
206 }
207 
208 // Branch analysis.
209 bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
210                                      MachineBasicBlock *&TBB,
211                                      MachineBasicBlock *&FBB,
212                                      SmallVectorImpl<MachineOperand> &Cond,
213                                      bool AllowModify) const {
214   // If the block has no terminators, it just falls into the block after it.
215   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
216   if (I == MBB.end())
217     return false;
218 
219   if (!isUnpredicatedTerminator(*I))
220     return false;
221 
222   // Get the last instruction in the block.
223   MachineInstr *LastInst = &*I;
224 
225   // If there is only one terminator instruction, process it.
226   unsigned LastOpc = LastInst->getOpcode();
227   if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
228     if (isUncondBranchOpcode(LastOpc)) {
229       TBB = LastInst->getOperand(0).getMBB();
230       return false;
231     }
232     if (isCondBranchOpcode(LastOpc)) {
233       // Block ends with fall-through condbranch.
234       parseCondBranch(LastInst, TBB, Cond);
235       return false;
236     }
237     return true; // Can't handle indirect branch.
238   }
239 
240   // Get the instruction before it if it is a terminator.
241   MachineInstr *SecondLastInst = &*I;
242   unsigned SecondLastOpc = SecondLastInst->getOpcode();
243 
244   // If AllowModify is true and the block ends with two or more unconditional
245   // branches, delete all but the first unconditional branch.
246   if (AllowModify && isUncondBranchOpcode(LastOpc)) {
247     while (isUncondBranchOpcode(SecondLastOpc)) {
248       LastInst->eraseFromParent();
249       LastInst = SecondLastInst;
250       LastOpc = LastInst->getOpcode();
251       if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
252         // Return now the only terminator is an unconditional branch.
253         TBB = LastInst->getOperand(0).getMBB();
254         return false;
255       } else {
256         SecondLastInst = &*I;
257         SecondLastOpc = SecondLastInst->getOpcode();
258       }
259     }
260   }
261 
262   // If there are three terminators, we don't know what sort of block this is.
263   if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
264     return true;
265 
266   // If the block ends with a B and a Bcc, handle it.
267   if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
268     parseCondBranch(SecondLastInst, TBB, Cond);
269     FBB = LastInst->getOperand(0).getMBB();
270     return false;
271   }
272 
273   // If the block ends with two unconditional branches, handle it.  The second
274   // one is not executed, so remove it.
275   if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
276     TBB = SecondLastInst->getOperand(0).getMBB();
277     I = LastInst;
278     if (AllowModify)
279       I->eraseFromParent();
280     return false;
281   }
282 
283   // ...likewise if it ends with an indirect branch followed by an unconditional
284   // branch.
285   if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
286     I = LastInst;
287     if (AllowModify)
288       I->eraseFromParent();
289     return true;
290   }
291 
292   // Otherwise, can't handle this.
293   return true;
294 }
295 
296 bool AArch64InstrInfo::reverseBranchCondition(
297     SmallVectorImpl<MachineOperand> &Cond) const {
298   if (Cond[0].getImm() != -1) {
299     // Regular Bcc
300     AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
301     Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
302   } else {
303     // Folded compare-and-branch
304     switch (Cond[1].getImm()) {
305     default:
306       llvm_unreachable("Unknown conditional branch!");
307     case AArch64::CBZW:
308       Cond[1].setImm(AArch64::CBNZW);
309       break;
310     case AArch64::CBNZW:
311       Cond[1].setImm(AArch64::CBZW);
312       break;
313     case AArch64::CBZX:
314       Cond[1].setImm(AArch64::CBNZX);
315       break;
316     case AArch64::CBNZX:
317       Cond[1].setImm(AArch64::CBZX);
318       break;
319     case AArch64::TBZW:
320       Cond[1].setImm(AArch64::TBNZW);
321       break;
322     case AArch64::TBNZW:
323       Cond[1].setImm(AArch64::TBZW);
324       break;
325     case AArch64::TBZX:
326       Cond[1].setImm(AArch64::TBNZX);
327       break;
328     case AArch64::TBNZX:
329       Cond[1].setImm(AArch64::TBZX);
330       break;
331     }
332   }
333 
334   return false;
335 }
336 
337 unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
338                                         int *BytesRemoved) const {
339   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
340   if (I == MBB.end())
341     return 0;
342 
343   if (!isUncondBranchOpcode(I->getOpcode()) &&
344       !isCondBranchOpcode(I->getOpcode()))
345     return 0;
346 
347   // Remove the branch.
348   I->eraseFromParent();
349 
350   I = MBB.end();
351 
352   if (I == MBB.begin()) {
353     if (BytesRemoved)
354       *BytesRemoved = 4;
355     return 1;
356   }
357   --I;
358   if (!isCondBranchOpcode(I->getOpcode())) {
359     if (BytesRemoved)
360       *BytesRemoved = 4;
361     return 1;
362   }
363 
364   // Remove the branch.
365   I->eraseFromParent();
366   if (BytesRemoved)
367     *BytesRemoved = 8;
368 
369   return 2;
370 }
371 
372 void AArch64InstrInfo::instantiateCondBranch(
373     MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
374     ArrayRef<MachineOperand> Cond) const {
375   if (Cond[0].getImm() != -1) {
376     // Regular Bcc
377     BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
378   } else {
379     // Folded compare-and-branch
380     // Note that we use addOperand instead of addReg to keep the flags.
381     const MachineInstrBuilder MIB =
382         BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
383     if (Cond.size() > 3)
384       MIB.addImm(Cond[3].getImm());
385     MIB.addMBB(TBB);
386   }
387 }
388 
389 unsigned AArch64InstrInfo::insertBranch(
390     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
391     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
392   // Shouldn't be a fall through.
393   assert(TBB && "insertBranch must not be told to insert a fallthrough");
394 
395   if (!FBB) {
396     if (Cond.empty()) // Unconditional branch?
397       BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
398     else
399       instantiateCondBranch(MBB, DL, TBB, Cond);
400 
401     if (BytesAdded)
402       *BytesAdded = 4;
403 
404     return 1;
405   }
406 
407   // Two-way conditional branch.
408   instantiateCondBranch(MBB, DL, TBB, Cond);
409   BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
410 
411   if (BytesAdded)
412     *BytesAdded = 8;
413 
414   return 2;
415 }
416 
417 // Find the original register that VReg is copied from.
418 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
419   while (Register::isVirtualRegister(VReg)) {
420     const MachineInstr *DefMI = MRI.getVRegDef(VReg);
421     if (!DefMI->isFullCopy())
422       return VReg;
423     VReg = DefMI->getOperand(1).getReg();
424   }
425   return VReg;
426 }
427 
428 // Determine if VReg is defined by an instruction that can be folded into a
429 // csel instruction. If so, return the folded opcode, and the replacement
430 // register.
431 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
432                                 unsigned *NewVReg = nullptr) {
433   VReg = removeCopies(MRI, VReg);
434   if (!Register::isVirtualRegister(VReg))
435     return 0;
436 
437   bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
438   const MachineInstr *DefMI = MRI.getVRegDef(VReg);
439   unsigned Opc = 0;
440   unsigned SrcOpNum = 0;
441   switch (DefMI->getOpcode()) {
442   case AArch64::ADDSXri:
443   case AArch64::ADDSWri:
444     // if NZCV is used, do not fold.
445     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
446       return 0;
447     // fall-through to ADDXri and ADDWri.
448     LLVM_FALLTHROUGH;
449   case AArch64::ADDXri:
450   case AArch64::ADDWri:
451     // add x, 1 -> csinc.
452     if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
453         DefMI->getOperand(3).getImm() != 0)
454       return 0;
455     SrcOpNum = 1;
456     Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
457     break;
458 
459   case AArch64::ORNXrr:
460   case AArch64::ORNWrr: {
461     // not x -> csinv, represented as orn dst, xzr, src.
462     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
463     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
464       return 0;
465     SrcOpNum = 2;
466     Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
467     break;
468   }
469 
470   case AArch64::SUBSXrr:
471   case AArch64::SUBSWrr:
472     // if NZCV is used, do not fold.
473     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
474       return 0;
475     // fall-through to SUBXrr and SUBWrr.
476     LLVM_FALLTHROUGH;
477   case AArch64::SUBXrr:
478   case AArch64::SUBWrr: {
479     // neg x -> csneg, represented as sub dst, xzr, src.
480     unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
481     if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
482       return 0;
483     SrcOpNum = 2;
484     Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
485     break;
486   }
487   default:
488     return 0;
489   }
490   assert(Opc && SrcOpNum && "Missing parameters");
491 
492   if (NewVReg)
493     *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
494   return Opc;
495 }
496 
497 bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
498                                        ArrayRef<MachineOperand> Cond,
499                                        unsigned DstReg, unsigned TrueReg,
500                                        unsigned FalseReg, int &CondCycles,
501                                        int &TrueCycles,
502                                        int &FalseCycles) const {
503   // Check register classes.
504   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
505   const TargetRegisterClass *RC =
506       RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
507   if (!RC)
508     return false;
509 
510   // Also need to check the dest regclass, in case we're trying to optimize
511   // something like:
512   // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
513   if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
514     return false;
515 
516   // Expanding cbz/tbz requires an extra cycle of latency on the condition.
517   unsigned ExtraCondLat = Cond.size() != 1;
518 
519   // GPRs are handled by csel.
520   // FIXME: Fold in x+1, -x, and ~x when applicable.
521   if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
522       AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
523     // Single-cycle csel, csinc, csinv, and csneg.
524     CondCycles = 1 + ExtraCondLat;
525     TrueCycles = FalseCycles = 1;
526     if (canFoldIntoCSel(MRI, TrueReg))
527       TrueCycles = 0;
528     else if (canFoldIntoCSel(MRI, FalseReg))
529       FalseCycles = 0;
530     return true;
531   }
532 
533   // Scalar floating point is handled by fcsel.
534   // FIXME: Form fabs, fmin, and fmax when applicable.
535   if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
536       AArch64::FPR32RegClass.hasSubClassEq(RC)) {
537     CondCycles = 5 + ExtraCondLat;
538     TrueCycles = FalseCycles = 2;
539     return true;
540   }
541 
542   // Can't do vectors.
543   return false;
544 }
545 
546 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
547                                     MachineBasicBlock::iterator I,
548                                     const DebugLoc &DL, unsigned DstReg,
549                                     ArrayRef<MachineOperand> Cond,
550                                     unsigned TrueReg, unsigned FalseReg) const {
551   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
552 
553   // Parse the condition code, see parseCondBranch() above.
554   AArch64CC::CondCode CC;
555   switch (Cond.size()) {
556   default:
557     llvm_unreachable("Unknown condition opcode in Cond");
558   case 1: // b.cc
559     CC = AArch64CC::CondCode(Cond[0].getImm());
560     break;
561   case 3: { // cbz/cbnz
562     // We must insert a compare against 0.
563     bool Is64Bit;
564     switch (Cond[1].getImm()) {
565     default:
566       llvm_unreachable("Unknown branch opcode in Cond");
567     case AArch64::CBZW:
568       Is64Bit = false;
569       CC = AArch64CC::EQ;
570       break;
571     case AArch64::CBZX:
572       Is64Bit = true;
573       CC = AArch64CC::EQ;
574       break;
575     case AArch64::CBNZW:
576       Is64Bit = false;
577       CC = AArch64CC::NE;
578       break;
579     case AArch64::CBNZX:
580       Is64Bit = true;
581       CC = AArch64CC::NE;
582       break;
583     }
584     Register SrcReg = Cond[2].getReg();
585     if (Is64Bit) {
586       // cmp reg, #0 is actually subs xzr, reg, #0.
587       MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
588       BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
589           .addReg(SrcReg)
590           .addImm(0)
591           .addImm(0);
592     } else {
593       MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
594       BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
595           .addReg(SrcReg)
596           .addImm(0)
597           .addImm(0);
598     }
599     break;
600   }
601   case 4: { // tbz/tbnz
602     // We must insert a tst instruction.
603     switch (Cond[1].getImm()) {
604     default:
605       llvm_unreachable("Unknown branch opcode in Cond");
606     case AArch64::TBZW:
607     case AArch64::TBZX:
608       CC = AArch64CC::EQ;
609       break;
610     case AArch64::TBNZW:
611     case AArch64::TBNZX:
612       CC = AArch64CC::NE;
613       break;
614     }
615     // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
616     if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
617       BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
618           .addReg(Cond[2].getReg())
619           .addImm(
620               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
621     else
622       BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
623           .addReg(Cond[2].getReg())
624           .addImm(
625               AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
626     break;
627   }
628   }
629 
630   unsigned Opc = 0;
631   const TargetRegisterClass *RC = nullptr;
632   bool TryFold = false;
633   if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
634     RC = &AArch64::GPR64RegClass;
635     Opc = AArch64::CSELXr;
636     TryFold = true;
637   } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
638     RC = &AArch64::GPR32RegClass;
639     Opc = AArch64::CSELWr;
640     TryFold = true;
641   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
642     RC = &AArch64::FPR64RegClass;
643     Opc = AArch64::FCSELDrrr;
644   } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
645     RC = &AArch64::FPR32RegClass;
646     Opc = AArch64::FCSELSrrr;
647   }
648   assert(RC && "Unsupported regclass");
649 
650   // Try folding simple instructions into the csel.
651   if (TryFold) {
652     unsigned NewVReg = 0;
653     unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
654     if (FoldedOpc) {
655       // The folded opcodes csinc, csinc and csneg apply the operation to
656       // FalseReg, so we need to invert the condition.
657       CC = AArch64CC::getInvertedCondCode(CC);
658       TrueReg = FalseReg;
659     } else
660       FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
661 
662     // Fold the operation. Leave any dead instructions for DCE to clean up.
663     if (FoldedOpc) {
664       FalseReg = NewVReg;
665       Opc = FoldedOpc;
666       // The extends the live range of NewVReg.
667       MRI.clearKillFlags(NewVReg);
668     }
669   }
670 
671   // Pull all virtual register into the appropriate class.
672   MRI.constrainRegClass(TrueReg, RC);
673   MRI.constrainRegClass(FalseReg, RC);
674 
675   // Insert the csel.
676   BuildMI(MBB, I, DL, get(Opc), DstReg)
677       .addReg(TrueReg)
678       .addReg(FalseReg)
679       .addImm(CC);
680 }
681 
682 /// Returns true if a MOVi32imm or MOVi64imm can be expanded to an  ORRxx.
683 static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
684   uint64_t Imm = MI.getOperand(1).getImm();
685   uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
686   uint64_t Encoding;
687   return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
688 }
689 
690 // FIXME: this implementation should be micro-architecture dependent, so a
691 // micro-architecture target hook should be introduced here in future.
692 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
693   if (!Subtarget.hasCustomCheapAsMoveHandling())
694     return MI.isAsCheapAsAMove();
695 
696   const unsigned Opcode = MI.getOpcode();
697 
698   // Firstly, check cases gated by features.
699 
700   if (Subtarget.hasZeroCycleZeroingFP()) {
701     if (Opcode == AArch64::FMOVH0 ||
702         Opcode == AArch64::FMOVS0 ||
703         Opcode == AArch64::FMOVD0)
704       return true;
705   }
706 
707   if (Subtarget.hasZeroCycleZeroingGP()) {
708     if (Opcode == TargetOpcode::COPY &&
709         (MI.getOperand(1).getReg() == AArch64::WZR ||
710          MI.getOperand(1).getReg() == AArch64::XZR))
711       return true;
712   }
713 
714   // Secondly, check cases specific to sub-targets.
715 
716   if (Subtarget.hasExynosCheapAsMoveHandling()) {
717     if (isExynosCheapAsMove(MI))
718       return true;
719 
720     return MI.isAsCheapAsAMove();
721   }
722 
723   // Finally, check generic cases.
724 
725   switch (Opcode) {
726   default:
727     return false;
728 
729   // add/sub on register without shift
730   case AArch64::ADDWri:
731   case AArch64::ADDXri:
732   case AArch64::SUBWri:
733   case AArch64::SUBXri:
734     return (MI.getOperand(3).getImm() == 0);
735 
736   // logical ops on immediate
737   case AArch64::ANDWri:
738   case AArch64::ANDXri:
739   case AArch64::EORWri:
740   case AArch64::EORXri:
741   case AArch64::ORRWri:
742   case AArch64::ORRXri:
743     return true;
744 
745   // logical ops on register without shift
746   case AArch64::ANDWrr:
747   case AArch64::ANDXrr:
748   case AArch64::BICWrr:
749   case AArch64::BICXrr:
750   case AArch64::EONWrr:
751   case AArch64::EONXrr:
752   case AArch64::EORWrr:
753   case AArch64::EORXrr:
754   case AArch64::ORNWrr:
755   case AArch64::ORNXrr:
756   case AArch64::ORRWrr:
757   case AArch64::ORRXrr:
758     return true;
759 
760   // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
761   // ORRXri, it is as cheap as MOV
762   case AArch64::MOVi32imm:
763     return canBeExpandedToORR(MI, 32);
764   case AArch64::MOVi64imm:
765     return canBeExpandedToORR(MI, 64);
766   }
767 
768   llvm_unreachable("Unknown opcode to check as cheap as a move!");
769 }
770 
771 bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
772   switch (MI.getOpcode()) {
773   default:
774     return false;
775 
776   case AArch64::ADDWrs:
777   case AArch64::ADDXrs:
778   case AArch64::ADDSWrs:
779   case AArch64::ADDSXrs: {
780     unsigned Imm = MI.getOperand(3).getImm();
781     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
782     if (ShiftVal == 0)
783       return true;
784     return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
785   }
786 
787   case AArch64::ADDWrx:
788   case AArch64::ADDXrx:
789   case AArch64::ADDXrx64:
790   case AArch64::ADDSWrx:
791   case AArch64::ADDSXrx:
792   case AArch64::ADDSXrx64: {
793     unsigned Imm = MI.getOperand(3).getImm();
794     switch (AArch64_AM::getArithExtendType(Imm)) {
795     default:
796       return false;
797     case AArch64_AM::UXTB:
798     case AArch64_AM::UXTH:
799     case AArch64_AM::UXTW:
800     case AArch64_AM::UXTX:
801       return AArch64_AM::getArithShiftValue(Imm) <= 4;
802     }
803   }
804 
805   case AArch64::SUBWrs:
806   case AArch64::SUBSWrs: {
807     unsigned Imm = MI.getOperand(3).getImm();
808     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
809     return ShiftVal == 0 ||
810            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
811   }
812 
813   case AArch64::SUBXrs:
814   case AArch64::SUBSXrs: {
815     unsigned Imm = MI.getOperand(3).getImm();
816     unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
817     return ShiftVal == 0 ||
818            (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
819   }
820 
821   case AArch64::SUBWrx:
822   case AArch64::SUBXrx:
823   case AArch64::SUBXrx64:
824   case AArch64::SUBSWrx:
825   case AArch64::SUBSXrx:
826   case AArch64::SUBSXrx64: {
827     unsigned Imm = MI.getOperand(3).getImm();
828     switch (AArch64_AM::getArithExtendType(Imm)) {
829     default:
830       return false;
831     case AArch64_AM::UXTB:
832     case AArch64_AM::UXTH:
833     case AArch64_AM::UXTW:
834     case AArch64_AM::UXTX:
835       return AArch64_AM::getArithShiftValue(Imm) == 0;
836     }
837   }
838 
839   case AArch64::LDRBBroW:
840   case AArch64::LDRBBroX:
841   case AArch64::LDRBroW:
842   case AArch64::LDRBroX:
843   case AArch64::LDRDroW:
844   case AArch64::LDRDroX:
845   case AArch64::LDRHHroW:
846   case AArch64::LDRHHroX:
847   case AArch64::LDRHroW:
848   case AArch64::LDRHroX:
849   case AArch64::LDRQroW:
850   case AArch64::LDRQroX:
851   case AArch64::LDRSBWroW:
852   case AArch64::LDRSBWroX:
853   case AArch64::LDRSBXroW:
854   case AArch64::LDRSBXroX:
855   case AArch64::LDRSHWroW:
856   case AArch64::LDRSHWroX:
857   case AArch64::LDRSHXroW:
858   case AArch64::LDRSHXroX:
859   case AArch64::LDRSWroW:
860   case AArch64::LDRSWroX:
861   case AArch64::LDRSroW:
862   case AArch64::LDRSroX:
863   case AArch64::LDRWroW:
864   case AArch64::LDRWroX:
865   case AArch64::LDRXroW:
866   case AArch64::LDRXroX:
867   case AArch64::PRFMroW:
868   case AArch64::PRFMroX:
869   case AArch64::STRBBroW:
870   case AArch64::STRBBroX:
871   case AArch64::STRBroW:
872   case AArch64::STRBroX:
873   case AArch64::STRDroW:
874   case AArch64::STRDroX:
875   case AArch64::STRHHroW:
876   case AArch64::STRHHroX:
877   case AArch64::STRHroW:
878   case AArch64::STRHroX:
879   case AArch64::STRQroW:
880   case AArch64::STRQroX:
881   case AArch64::STRSroW:
882   case AArch64::STRSroX:
883   case AArch64::STRWroW:
884   case AArch64::STRWroX:
885   case AArch64::STRXroW:
886   case AArch64::STRXroX: {
887     unsigned IsSigned = MI.getOperand(3).getImm();
888     return !IsSigned;
889   }
890   }
891 }
892 
893 bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
894   unsigned Opc = MI.getOpcode();
895   switch (Opc) {
896     default:
897       return false;
898     case AArch64::SEH_StackAlloc:
899     case AArch64::SEH_SaveFPLR:
900     case AArch64::SEH_SaveFPLR_X:
901     case AArch64::SEH_SaveReg:
902     case AArch64::SEH_SaveReg_X:
903     case AArch64::SEH_SaveRegP:
904     case AArch64::SEH_SaveRegP_X:
905     case AArch64::SEH_SaveFReg:
906     case AArch64::SEH_SaveFReg_X:
907     case AArch64::SEH_SaveFRegP:
908     case AArch64::SEH_SaveFRegP_X:
909     case AArch64::SEH_SetFP:
910     case AArch64::SEH_AddFP:
911     case AArch64::SEH_Nop:
912     case AArch64::SEH_PrologEnd:
913     case AArch64::SEH_EpilogStart:
914     case AArch64::SEH_EpilogEnd:
915       return true;
916   }
917 }
918 
919 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
920                                              unsigned &SrcReg, unsigned &DstReg,
921                                              unsigned &SubIdx) const {
922   switch (MI.getOpcode()) {
923   default:
924     return false;
925   case AArch64::SBFMXri: // aka sxtw
926   case AArch64::UBFMXri: // aka uxtw
927     // Check for the 32 -> 64 bit extension case, these instructions can do
928     // much more.
929     if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
930       return false;
931     // This is a signed or unsigned 32 -> 64 bit extension.
932     SrcReg = MI.getOperand(1).getReg();
933     DstReg = MI.getOperand(0).getReg();
934     SubIdx = AArch64::sub_32;
935     return true;
936   }
937 }
938 
939 bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
940     const MachineInstr &MIa, const MachineInstr &MIb) const {
941   const TargetRegisterInfo *TRI = &getRegisterInfo();
942   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
943   int64_t OffsetA = 0, OffsetB = 0;
944   unsigned WidthA = 0, WidthB = 0;
945 
946   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
947   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
948 
949   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
950       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
951     return false;
952 
953   // Retrieve the base, offset from the base and width. Width
954   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8).  If
955   // base are identical, and the offset of a lower memory access +
956   // the width doesn't overlap the offset of a higher memory access,
957   // then the memory accesses are different.
958   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
959       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
960     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
961       int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
962       int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
963       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
964       if (LowOffset + LowWidth <= HighOffset)
965         return true;
966     }
967   }
968   return false;
969 }
970 
971 bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
972                                             const MachineBasicBlock *MBB,
973                                             const MachineFunction &MF) const {
974   if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
975     return true;
976   switch (MI.getOpcode()) {
977   case AArch64::HINT:
978     // CSDB hints are scheduling barriers.
979     if (MI.getOperand(0).getImm() == 0x14)
980       return true;
981     break;
982   case AArch64::DSB:
983   case AArch64::ISB:
984     // DSB and ISB also are scheduling barriers.
985     return true;
986   default:;
987   }
988   return isSEHInstruction(MI);
989 }
990 
991 /// analyzeCompare - For a comparison instruction, return the source registers
992 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
993 /// Return true if the comparison instruction can be analyzed.
994 bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
995                                       unsigned &SrcReg2, int &CmpMask,
996                                       int &CmpValue) const {
997   // The first operand can be a frame index where we'd normally expect a
998   // register.
999   assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1000   if (!MI.getOperand(1).isReg())
1001     return false;
1002 
1003   switch (MI.getOpcode()) {
1004   default:
1005     break;
1006   case AArch64::SUBSWrr:
1007   case AArch64::SUBSWrs:
1008   case AArch64::SUBSWrx:
1009   case AArch64::SUBSXrr:
1010   case AArch64::SUBSXrs:
1011   case AArch64::SUBSXrx:
1012   case AArch64::ADDSWrr:
1013   case AArch64::ADDSWrs:
1014   case AArch64::ADDSWrx:
1015   case AArch64::ADDSXrr:
1016   case AArch64::ADDSXrs:
1017   case AArch64::ADDSXrx:
1018     // Replace SUBSWrr with SUBWrr if NZCV is not used.
1019     SrcReg = MI.getOperand(1).getReg();
1020     SrcReg2 = MI.getOperand(2).getReg();
1021     CmpMask = ~0;
1022     CmpValue = 0;
1023     return true;
1024   case AArch64::SUBSWri:
1025   case AArch64::ADDSWri:
1026   case AArch64::SUBSXri:
1027   case AArch64::ADDSXri:
1028     SrcReg = MI.getOperand(1).getReg();
1029     SrcReg2 = 0;
1030     CmpMask = ~0;
1031     // FIXME: In order to convert CmpValue to 0 or 1
1032     CmpValue = MI.getOperand(2).getImm() != 0;
1033     return true;
1034   case AArch64::ANDSWri:
1035   case AArch64::ANDSXri:
1036     // ANDS does not use the same encoding scheme as the others xxxS
1037     // instructions.
1038     SrcReg = MI.getOperand(1).getReg();
1039     SrcReg2 = 0;
1040     CmpMask = ~0;
1041     // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
1042     // while the type of CmpValue is int. When converting uint64_t to int,
1043     // the high 32 bits of uint64_t will be lost.
1044     // In fact it causes a bug in spec2006-483.xalancbmk
1045     // CmpValue is only used to compare with zero in OptimizeCompareInstr
1046     CmpValue = AArch64_AM::decodeLogicalImmediate(
1047                    MI.getOperand(2).getImm(),
1048                    MI.getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0;
1049     return true;
1050   }
1051 
1052   return false;
1053 }
1054 
1055 static bool UpdateOperandRegClass(MachineInstr &Instr) {
1056   MachineBasicBlock *MBB = Instr.getParent();
1057   assert(MBB && "Can't get MachineBasicBlock here");
1058   MachineFunction *MF = MBB->getParent();
1059   assert(MF && "Can't get MachineFunction here");
1060   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1061   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1062   MachineRegisterInfo *MRI = &MF->getRegInfo();
1063 
1064   for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1065        ++OpIdx) {
1066     MachineOperand &MO = Instr.getOperand(OpIdx);
1067     const TargetRegisterClass *OpRegCstraints =
1068         Instr.getRegClassConstraint(OpIdx, TII, TRI);
1069 
1070     // If there's no constraint, there's nothing to do.
1071     if (!OpRegCstraints)
1072       continue;
1073     // If the operand is a frame index, there's nothing to do here.
1074     // A frame index operand will resolve correctly during PEI.
1075     if (MO.isFI())
1076       continue;
1077 
1078     assert(MO.isReg() &&
1079            "Operand has register constraints without being a register!");
1080 
1081     Register Reg = MO.getReg();
1082     if (Register::isPhysicalRegister(Reg)) {
1083       if (!OpRegCstraints->contains(Reg))
1084         return false;
1085     } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
1086                !MRI->constrainRegClass(Reg, OpRegCstraints))
1087       return false;
1088   }
1089 
1090   return true;
1091 }
1092 
1093 /// Return the opcode that does not set flags when possible - otherwise
1094 /// return the original opcode. The caller is responsible to do the actual
1095 /// substitution and legality checking.
1096 static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1097   // Don't convert all compare instructions, because for some the zero register
1098   // encoding becomes the sp register.
1099   bool MIDefinesZeroReg = false;
1100   if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR))
1101     MIDefinesZeroReg = true;
1102 
1103   switch (MI.getOpcode()) {
1104   default:
1105     return MI.getOpcode();
1106   case AArch64::ADDSWrr:
1107     return AArch64::ADDWrr;
1108   case AArch64::ADDSWri:
1109     return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1110   case AArch64::ADDSWrs:
1111     return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1112   case AArch64::ADDSWrx:
1113     return AArch64::ADDWrx;
1114   case AArch64::ADDSXrr:
1115     return AArch64::ADDXrr;
1116   case AArch64::ADDSXri:
1117     return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1118   case AArch64::ADDSXrs:
1119     return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1120   case AArch64::ADDSXrx:
1121     return AArch64::ADDXrx;
1122   case AArch64::SUBSWrr:
1123     return AArch64::SUBWrr;
1124   case AArch64::SUBSWri:
1125     return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1126   case AArch64::SUBSWrs:
1127     return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1128   case AArch64::SUBSWrx:
1129     return AArch64::SUBWrx;
1130   case AArch64::SUBSXrr:
1131     return AArch64::SUBXrr;
1132   case AArch64::SUBSXri:
1133     return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1134   case AArch64::SUBSXrs:
1135     return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1136   case AArch64::SUBSXrx:
1137     return AArch64::SUBXrx;
1138   }
1139 }
1140 
1141 enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1142 
1143 /// True when condition flags are accessed (either by writing or reading)
1144 /// on the instruction trace starting at From and ending at To.
1145 ///
1146 /// Note: If From and To are from different blocks it's assumed CC are accessed
1147 ///       on the path.
1148 static bool areCFlagsAccessedBetweenInstrs(
1149     MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1150     const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1151   // Early exit if To is at the beginning of the BB.
1152   if (To == To->getParent()->begin())
1153     return true;
1154 
1155   // Check whether the instructions are in the same basic block
1156   // If not, assume the condition flags might get modified somewhere.
1157   if (To->getParent() != From->getParent())
1158     return true;
1159 
1160   // From must be above To.
1161   assert(std::find_if(++To.getReverse(), To->getParent()->rend(),
1162                       [From](MachineInstr &MI) {
1163                         return MI.getIterator() == From;
1164                       }) != To->getParent()->rend());
1165 
1166   // We iterate backward starting \p To until we hit \p From.
1167   for (--To; To != From; --To) {
1168     const MachineInstr &Instr = *To;
1169 
1170     if (((AccessToCheck & AK_Write) &&
1171          Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
1172         ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
1173       return true;
1174   }
1175   return false;
1176 }
1177 
1178 /// Try to optimize a compare instruction. A compare instruction is an
1179 /// instruction which produces AArch64::NZCV. It can be truly compare
1180 /// instruction
1181 /// when there are no uses of its destination register.
1182 ///
1183 /// The following steps are tried in order:
1184 /// 1. Convert CmpInstr into an unconditional version.
1185 /// 2. Remove CmpInstr if above there is an instruction producing a needed
1186 ///    condition code or an instruction which can be converted into such an
1187 ///    instruction.
1188 ///    Only comparison with zero is supported.
1189 bool AArch64InstrInfo::optimizeCompareInstr(
1190     MachineInstr &CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
1191     int CmpValue, const MachineRegisterInfo *MRI) const {
1192   assert(CmpInstr.getParent());
1193   assert(MRI);
1194 
1195   // Replace SUBSWrr with SUBWrr if NZCV is not used.
1196   int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true);
1197   if (DeadNZCVIdx != -1) {
1198     if (CmpInstr.definesRegister(AArch64::WZR) ||
1199         CmpInstr.definesRegister(AArch64::XZR)) {
1200       CmpInstr.eraseFromParent();
1201       return true;
1202     }
1203     unsigned Opc = CmpInstr.getOpcode();
1204     unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
1205     if (NewOpc == Opc)
1206       return false;
1207     const MCInstrDesc &MCID = get(NewOpc);
1208     CmpInstr.setDesc(MCID);
1209     CmpInstr.RemoveOperand(DeadNZCVIdx);
1210     bool succeeded = UpdateOperandRegClass(CmpInstr);
1211     (void)succeeded;
1212     assert(succeeded && "Some operands reg class are incompatible!");
1213     return true;
1214   }
1215 
1216   // Continue only if we have a "ri" where immediate is zero.
1217   // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
1218   // function.
1219   assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
1220   if (CmpValue != 0 || SrcReg2 != 0)
1221     return false;
1222 
1223   // CmpInstr is a Compare instruction if destination register is not used.
1224   if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
1225     return false;
1226 
1227   return substituteCmpToZero(CmpInstr, SrcReg, MRI);
1228 }
1229 
1230 /// Get opcode of S version of Instr.
1231 /// If Instr is S version its opcode is returned.
1232 /// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
1233 /// or we are not interested in it.
1234 static unsigned sForm(MachineInstr &Instr) {
1235   switch (Instr.getOpcode()) {
1236   default:
1237     return AArch64::INSTRUCTION_LIST_END;
1238 
1239   case AArch64::ADDSWrr:
1240   case AArch64::ADDSWri:
1241   case AArch64::ADDSXrr:
1242   case AArch64::ADDSXri:
1243   case AArch64::SUBSWrr:
1244   case AArch64::SUBSWri:
1245   case AArch64::SUBSXrr:
1246   case AArch64::SUBSXri:
1247     return Instr.getOpcode();
1248 
1249   case AArch64::ADDWrr:
1250     return AArch64::ADDSWrr;
1251   case AArch64::ADDWri:
1252     return AArch64::ADDSWri;
1253   case AArch64::ADDXrr:
1254     return AArch64::ADDSXrr;
1255   case AArch64::ADDXri:
1256     return AArch64::ADDSXri;
1257   case AArch64::ADCWr:
1258     return AArch64::ADCSWr;
1259   case AArch64::ADCXr:
1260     return AArch64::ADCSXr;
1261   case AArch64::SUBWrr:
1262     return AArch64::SUBSWrr;
1263   case AArch64::SUBWri:
1264     return AArch64::SUBSWri;
1265   case AArch64::SUBXrr:
1266     return AArch64::SUBSXrr;
1267   case AArch64::SUBXri:
1268     return AArch64::SUBSXri;
1269   case AArch64::SBCWr:
1270     return AArch64::SBCSWr;
1271   case AArch64::SBCXr:
1272     return AArch64::SBCSXr;
1273   case AArch64::ANDWri:
1274     return AArch64::ANDSWri;
1275   case AArch64::ANDXri:
1276     return AArch64::ANDSXri;
1277   }
1278 }
1279 
1280 /// Check if AArch64::NZCV should be alive in successors of MBB.
1281 static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) {
1282   for (auto *BB : MBB->successors())
1283     if (BB->isLiveIn(AArch64::NZCV))
1284       return true;
1285   return false;
1286 }
1287 
1288 namespace {
1289 
1290 struct UsedNZCV {
1291   bool N = false;
1292   bool Z = false;
1293   bool C = false;
1294   bool V = false;
1295 
1296   UsedNZCV() = default;
1297 
1298   UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
1299     this->N |= UsedFlags.N;
1300     this->Z |= UsedFlags.Z;
1301     this->C |= UsedFlags.C;
1302     this->V |= UsedFlags.V;
1303     return *this;
1304   }
1305 };
1306 
1307 } // end anonymous namespace
1308 
1309 /// Find a condition code used by the instruction.
1310 /// Returns AArch64CC::Invalid if either the instruction does not use condition
1311 /// codes or we don't optimize CmpInstr in the presence of such instructions.
1312 static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
1313   switch (Instr.getOpcode()) {
1314   default:
1315     return AArch64CC::Invalid;
1316 
1317   case AArch64::Bcc: {
1318     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1319     assert(Idx >= 2);
1320     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 2).getImm());
1321   }
1322 
1323   case AArch64::CSINVWr:
1324   case AArch64::CSINVXr:
1325   case AArch64::CSINCWr:
1326   case AArch64::CSINCXr:
1327   case AArch64::CSELWr:
1328   case AArch64::CSELXr:
1329   case AArch64::CSNEGWr:
1330   case AArch64::CSNEGXr:
1331   case AArch64::FCSELSrrr:
1332   case AArch64::FCSELDrrr: {
1333     int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1334     assert(Idx >= 1);
1335     return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 1).getImm());
1336   }
1337   }
1338 }
1339 
1340 static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
1341   assert(CC != AArch64CC::Invalid);
1342   UsedNZCV UsedFlags;
1343   switch (CC) {
1344   default:
1345     break;
1346 
1347   case AArch64CC::EQ: // Z set
1348   case AArch64CC::NE: // Z clear
1349     UsedFlags.Z = true;
1350     break;
1351 
1352   case AArch64CC::HI: // Z clear and C set
1353   case AArch64CC::LS: // Z set   or  C clear
1354     UsedFlags.Z = true;
1355     LLVM_FALLTHROUGH;
1356   case AArch64CC::HS: // C set
1357   case AArch64CC::LO: // C clear
1358     UsedFlags.C = true;
1359     break;
1360 
1361   case AArch64CC::MI: // N set
1362   case AArch64CC::PL: // N clear
1363     UsedFlags.N = true;
1364     break;
1365 
1366   case AArch64CC::VS: // V set
1367   case AArch64CC::VC: // V clear
1368     UsedFlags.V = true;
1369     break;
1370 
1371   case AArch64CC::GT: // Z clear, N and V the same
1372   case AArch64CC::LE: // Z set,   N and V differ
1373     UsedFlags.Z = true;
1374     LLVM_FALLTHROUGH;
1375   case AArch64CC::GE: // N and V the same
1376   case AArch64CC::LT: // N and V differ
1377     UsedFlags.N = true;
1378     UsedFlags.V = true;
1379     break;
1380   }
1381   return UsedFlags;
1382 }
1383 
1384 static bool isADDSRegImm(unsigned Opcode) {
1385   return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1386 }
1387 
1388 static bool isSUBSRegImm(unsigned Opcode) {
1389   return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1390 }
1391 
1392 /// Check if CmpInstr can be substituted by MI.
1393 ///
1394 /// CmpInstr can be substituted:
1395 /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1396 /// - and, MI and CmpInstr are from the same MachineBB
1397 /// - and, condition flags are not alive in successors of the CmpInstr parent
1398 /// - and, if MI opcode is the S form there must be no defs of flags between
1399 ///        MI and CmpInstr
1400 ///        or if MI opcode is not the S form there must be neither defs of flags
1401 ///        nor uses of flags between MI and CmpInstr.
1402 /// - and  C/V flags are not used after CmpInstr
1403 static bool canInstrSubstituteCmpInstr(MachineInstr *MI, MachineInstr *CmpInstr,
1404                                        const TargetRegisterInfo *TRI) {
1405   assert(MI);
1406   assert(sForm(*MI) != AArch64::INSTRUCTION_LIST_END);
1407   assert(CmpInstr);
1408 
1409   const unsigned CmpOpcode = CmpInstr->getOpcode();
1410   if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
1411     return false;
1412 
1413   if (MI->getParent() != CmpInstr->getParent())
1414     return false;
1415 
1416   if (areCFlagsAliveInSuccessors(CmpInstr->getParent()))
1417     return false;
1418 
1419   AccessKind AccessToCheck = AK_Write;
1420   if (sForm(*MI) != MI->getOpcode())
1421     AccessToCheck = AK_All;
1422   if (areCFlagsAccessedBetweenInstrs(MI, CmpInstr, TRI, AccessToCheck))
1423     return false;
1424 
1425   UsedNZCV NZCVUsedAfterCmp;
1426   for (auto I = std::next(CmpInstr->getIterator()),
1427             E = CmpInstr->getParent()->instr_end();
1428        I != E; ++I) {
1429     const MachineInstr &Instr = *I;
1430     if (Instr.readsRegister(AArch64::NZCV, TRI)) {
1431       AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
1432       if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
1433         return false;
1434       NZCVUsedAfterCmp |= getUsedNZCV(CC);
1435     }
1436 
1437     if (Instr.modifiesRegister(AArch64::NZCV, TRI))
1438       break;
1439   }
1440 
1441   return !NZCVUsedAfterCmp.C && !NZCVUsedAfterCmp.V;
1442 }
1443 
1444 /// Substitute an instruction comparing to zero with another instruction
1445 /// which produces needed condition flags.
1446 ///
1447 /// Return true on success.
1448 bool AArch64InstrInfo::substituteCmpToZero(
1449     MachineInstr &CmpInstr, unsigned SrcReg,
1450     const MachineRegisterInfo *MRI) const {
1451   assert(MRI);
1452   // Get the unique definition of SrcReg.
1453   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1454   if (!MI)
1455     return false;
1456 
1457   const TargetRegisterInfo *TRI = &getRegisterInfo();
1458 
1459   unsigned NewOpc = sForm(*MI);
1460   if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1461     return false;
1462 
1463   if (!canInstrSubstituteCmpInstr(MI, &CmpInstr, TRI))
1464     return false;
1465 
1466   // Update the instruction to set NZCV.
1467   MI->setDesc(get(NewOpc));
1468   CmpInstr.eraseFromParent();
1469   bool succeeded = UpdateOperandRegClass(*MI);
1470   (void)succeeded;
1471   assert(succeeded && "Some operands reg class are incompatible!");
1472   MI->addRegisterDefined(AArch64::NZCV, TRI);
1473   return true;
1474 }
1475 
1476 bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1477   if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1478       MI.getOpcode() != AArch64::CATCHRET)
1479     return false;
1480 
1481   MachineBasicBlock &MBB = *MI.getParent();
1482   auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1483   auto TRI = Subtarget.getRegisterInfo();
1484   DebugLoc DL = MI.getDebugLoc();
1485 
1486   if (MI.getOpcode() == AArch64::CATCHRET) {
1487     // Skip to the first instruction before the epilog.
1488     const TargetInstrInfo *TII =
1489       MBB.getParent()->getSubtarget().getInstrInfo();
1490     MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
1491     auto MBBI = MachineBasicBlock::iterator(MI);
1492     MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
1493     while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
1494            FirstEpilogSEH != MBB.begin())
1495       FirstEpilogSEH = std::prev(FirstEpilogSEH);
1496     if (FirstEpilogSEH != MBB.begin())
1497       FirstEpilogSEH = std::next(FirstEpilogSEH);
1498     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
1499         .addReg(AArch64::X0, RegState::Define)
1500         .addMBB(TargetMBB);
1501     BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
1502         .addReg(AArch64::X0, RegState::Define)
1503         .addReg(AArch64::X0)
1504         .addMBB(TargetMBB)
1505         .addImm(0);
1506     return true;
1507   }
1508 
1509   Register Reg = MI.getOperand(0).getReg();
1510   const GlobalValue *GV =
1511       cast<GlobalValue>((*MI.memoperands_begin())->getValue());
1512   const TargetMachine &TM = MBB.getParent()->getTarget();
1513   unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
1514   const unsigned char MO_NC = AArch64II::MO_NC;
1515 
1516   if ((OpFlags & AArch64II::MO_GOT) != 0) {
1517     BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
1518         .addGlobalAddress(GV, 0, OpFlags);
1519     if (Subtarget.isTargetILP32()) {
1520       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1521       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1522           .addDef(Reg32, RegState::Dead)
1523           .addUse(Reg, RegState::Kill)
1524           .addImm(0)
1525           .addMemOperand(*MI.memoperands_begin())
1526           .addDef(Reg, RegState::Implicit);
1527     } else {
1528       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1529           .addReg(Reg, RegState::Kill)
1530           .addImm(0)
1531           .addMemOperand(*MI.memoperands_begin());
1532     }
1533   } else if (TM.getCodeModel() == CodeModel::Large) {
1534     assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?");
1535     BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
1536         .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
1537         .addImm(0);
1538     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1539         .addReg(Reg, RegState::Kill)
1540         .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
1541         .addImm(16);
1542     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1543         .addReg(Reg, RegState::Kill)
1544         .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
1545         .addImm(32);
1546     BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
1547         .addReg(Reg, RegState::Kill)
1548         .addGlobalAddress(GV, 0, AArch64II::MO_G3)
1549         .addImm(48);
1550     BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1551         .addReg(Reg, RegState::Kill)
1552         .addImm(0)
1553         .addMemOperand(*MI.memoperands_begin());
1554   } else if (TM.getCodeModel() == CodeModel::Tiny) {
1555     BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg)
1556         .addGlobalAddress(GV, 0, OpFlags);
1557   } else {
1558     BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
1559         .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
1560     unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
1561     if (Subtarget.isTargetILP32()) {
1562       unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1563       BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1564           .addDef(Reg32, RegState::Dead)
1565           .addUse(Reg, RegState::Kill)
1566           .addGlobalAddress(GV, 0, LoFlags)
1567           .addMemOperand(*MI.memoperands_begin())
1568           .addDef(Reg, RegState::Implicit);
1569     } else {
1570       BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
1571           .addReg(Reg, RegState::Kill)
1572           .addGlobalAddress(GV, 0, LoFlags)
1573           .addMemOperand(*MI.memoperands_begin());
1574     }
1575   }
1576 
1577   MBB.erase(MI);
1578 
1579   return true;
1580 }
1581 
1582 // Return true if this instruction simply sets its single destination register
1583 // to zero. This is equivalent to a register rename of the zero-register.
1584 bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
1585   switch (MI.getOpcode()) {
1586   default:
1587     break;
1588   case AArch64::MOVZWi:
1589   case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1590     if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
1591       assert(MI.getDesc().getNumOperands() == 3 &&
1592              MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1593       return true;
1594     }
1595     break;
1596   case AArch64::ANDWri: // and Rd, Rzr, #imm
1597     return MI.getOperand(1).getReg() == AArch64::WZR;
1598   case AArch64::ANDXri:
1599     return MI.getOperand(1).getReg() == AArch64::XZR;
1600   case TargetOpcode::COPY:
1601     return MI.getOperand(1).getReg() == AArch64::WZR;
1602   }
1603   return false;
1604 }
1605 
1606 // Return true if this instruction simply renames a general register without
1607 // modifying bits.
1608 bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
1609   switch (MI.getOpcode()) {
1610   default:
1611     break;
1612   case TargetOpcode::COPY: {
1613     // GPR32 copies will by lowered to ORRXrs
1614     Register DstReg = MI.getOperand(0).getReg();
1615     return (AArch64::GPR32RegClass.contains(DstReg) ||
1616             AArch64::GPR64RegClass.contains(DstReg));
1617   }
1618   case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1619     if (MI.getOperand(1).getReg() == AArch64::XZR) {
1620       assert(MI.getDesc().getNumOperands() == 4 &&
1621              MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1622       return true;
1623     }
1624     break;
1625   case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1626     if (MI.getOperand(2).getImm() == 0) {
1627       assert(MI.getDesc().getNumOperands() == 4 &&
1628              MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1629       return true;
1630     }
1631     break;
1632   }
1633   return false;
1634 }
1635 
1636 // Return true if this instruction simply renames a general register without
1637 // modifying bits.
1638 bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
1639   switch (MI.getOpcode()) {
1640   default:
1641     break;
1642   case TargetOpcode::COPY: {
1643     // FPR64 copies will by lowered to ORR.16b
1644     Register DstReg = MI.getOperand(0).getReg();
1645     return (AArch64::FPR64RegClass.contains(DstReg) ||
1646             AArch64::FPR128RegClass.contains(DstReg));
1647   }
1648   case AArch64::ORRv16i8:
1649     if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
1650       assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
1651              "invalid ORRv16i8 operands");
1652       return true;
1653     }
1654     break;
1655   }
1656   return false;
1657 }
1658 
1659 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1660                                                int &FrameIndex) const {
1661   switch (MI.getOpcode()) {
1662   default:
1663     break;
1664   case AArch64::LDRWui:
1665   case AArch64::LDRXui:
1666   case AArch64::LDRBui:
1667   case AArch64::LDRHui:
1668   case AArch64::LDRSui:
1669   case AArch64::LDRDui:
1670   case AArch64::LDRQui:
1671     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1672         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1673       FrameIndex = MI.getOperand(1).getIndex();
1674       return MI.getOperand(0).getReg();
1675     }
1676     break;
1677   }
1678 
1679   return 0;
1680 }
1681 
1682 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1683                                               int &FrameIndex) const {
1684   switch (MI.getOpcode()) {
1685   default:
1686     break;
1687   case AArch64::STRWui:
1688   case AArch64::STRXui:
1689   case AArch64::STRBui:
1690   case AArch64::STRHui:
1691   case AArch64::STRSui:
1692   case AArch64::STRDui:
1693   case AArch64::STRQui:
1694   case AArch64::LDR_PXI:
1695   case AArch64::STR_PXI:
1696     if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
1697         MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
1698       FrameIndex = MI.getOperand(1).getIndex();
1699       return MI.getOperand(0).getReg();
1700     }
1701     break;
1702   }
1703   return 0;
1704 }
1705 
1706 /// Check all MachineMemOperands for a hint to suppress pairing.
1707 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
1708   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1709     return MMO->getFlags() & MOSuppressPair;
1710   });
1711 }
1712 
1713 /// Set a flag on the first MachineMemOperand to suppress pairing.
1714 void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
1715   if (MI.memoperands_empty())
1716     return;
1717   (*MI.memoperands_begin())->setFlags(MOSuppressPair);
1718 }
1719 
1720 /// Check all MachineMemOperands for a hint that the load/store is strided.
1721 bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
1722   return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
1723     return MMO->getFlags() & MOStridedAccess;
1724   });
1725 }
1726 
1727 bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) {
1728   switch (Opc) {
1729   default:
1730     return false;
1731   case AArch64::STURSi:
1732   case AArch64::STURDi:
1733   case AArch64::STURQi:
1734   case AArch64::STURBBi:
1735   case AArch64::STURHHi:
1736   case AArch64::STURWi:
1737   case AArch64::STURXi:
1738   case AArch64::LDURSi:
1739   case AArch64::LDURDi:
1740   case AArch64::LDURQi:
1741   case AArch64::LDURWi:
1742   case AArch64::LDURXi:
1743   case AArch64::LDURSWi:
1744   case AArch64::LDURHHi:
1745   case AArch64::LDURBBi:
1746   case AArch64::LDURSBWi:
1747   case AArch64::LDURSHWi:
1748     return true;
1749   }
1750 }
1751 
1752 Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
1753   switch (Opc) {
1754   default: return {};
1755   case AArch64::PRFMui: return AArch64::PRFUMi;
1756   case AArch64::LDRXui: return AArch64::LDURXi;
1757   case AArch64::LDRWui: return AArch64::LDURWi;
1758   case AArch64::LDRBui: return AArch64::LDURBi;
1759   case AArch64::LDRHui: return AArch64::LDURHi;
1760   case AArch64::LDRSui: return AArch64::LDURSi;
1761   case AArch64::LDRDui: return AArch64::LDURDi;
1762   case AArch64::LDRQui: return AArch64::LDURQi;
1763   case AArch64::LDRBBui: return AArch64::LDURBBi;
1764   case AArch64::LDRHHui: return AArch64::LDURHHi;
1765   case AArch64::LDRSBXui: return AArch64::LDURSBXi;
1766   case AArch64::LDRSBWui: return AArch64::LDURSBWi;
1767   case AArch64::LDRSHXui: return AArch64::LDURSHXi;
1768   case AArch64::LDRSHWui: return AArch64::LDURSHWi;
1769   case AArch64::LDRSWui: return AArch64::LDURSWi;
1770   case AArch64::STRXui: return AArch64::STURXi;
1771   case AArch64::STRWui: return AArch64::STURWi;
1772   case AArch64::STRBui: return AArch64::STURBi;
1773   case AArch64::STRHui: return AArch64::STURHi;
1774   case AArch64::STRSui: return AArch64::STURSi;
1775   case AArch64::STRDui: return AArch64::STURDi;
1776   case AArch64::STRQui: return AArch64::STURQi;
1777   case AArch64::STRBBui: return AArch64::STURBBi;
1778   case AArch64::STRHHui: return AArch64::STURHHi;
1779   }
1780 }
1781 
1782 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
1783   switch (Opc) {
1784   default:
1785     return 2;
1786   case AArch64::LDPXi:
1787   case AArch64::LDPDi:
1788   case AArch64::STPXi:
1789   case AArch64::STPDi:
1790   case AArch64::LDNPXi:
1791   case AArch64::LDNPDi:
1792   case AArch64::STNPXi:
1793   case AArch64::STNPDi:
1794   case AArch64::LDPQi:
1795   case AArch64::STPQi:
1796   case AArch64::LDNPQi:
1797   case AArch64::STNPQi:
1798   case AArch64::LDPWi:
1799   case AArch64::LDPSi:
1800   case AArch64::STPWi:
1801   case AArch64::STPSi:
1802   case AArch64::LDNPWi:
1803   case AArch64::LDNPSi:
1804   case AArch64::STNPWi:
1805   case AArch64::STNPSi:
1806   case AArch64::LDG:
1807   case AArch64::STGPi:
1808   case AArch64::LD1B_IMM:
1809   case AArch64::LD1H_IMM:
1810   case AArch64::LD1W_IMM:
1811   case AArch64::LD1D_IMM:
1812   case AArch64::ST1B_IMM:
1813   case AArch64::ST1H_IMM:
1814   case AArch64::ST1W_IMM:
1815   case AArch64::ST1D_IMM:
1816     return 3;
1817   case AArch64::ADDG:
1818   case AArch64::STGOffset:
1819   case AArch64::LDR_PXI:
1820   case AArch64::STR_PXI:
1821     return 2;
1822   }
1823 }
1824 
1825 bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
1826   switch (MI.getOpcode()) {
1827   default:
1828     return false;
1829   // Scaled instructions.
1830   case AArch64::STRSui:
1831   case AArch64::STRDui:
1832   case AArch64::STRQui:
1833   case AArch64::STRXui:
1834   case AArch64::STRWui:
1835   case AArch64::LDRSui:
1836   case AArch64::LDRDui:
1837   case AArch64::LDRQui:
1838   case AArch64::LDRXui:
1839   case AArch64::LDRWui:
1840   case AArch64::LDRSWui:
1841   // Unscaled instructions.
1842   case AArch64::STURSi:
1843   case AArch64::STURDi:
1844   case AArch64::STURQi:
1845   case AArch64::STURWi:
1846   case AArch64::STURXi:
1847   case AArch64::LDURSi:
1848   case AArch64::LDURDi:
1849   case AArch64::LDURQi:
1850   case AArch64::LDURWi:
1851   case AArch64::LDURXi:
1852   case AArch64::LDURSWi:
1853     return true;
1854   }
1855 }
1856 
1857 unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc,
1858                                                    bool &Is64Bit) {
1859   switch (Opc) {
1860   default:
1861     llvm_unreachable("Opcode has no flag setting equivalent!");
1862   // 32-bit cases:
1863   case AArch64::ADDWri:
1864     Is64Bit = false;
1865     return AArch64::ADDSWri;
1866   case AArch64::ADDWrr:
1867     Is64Bit = false;
1868     return AArch64::ADDSWrr;
1869   case AArch64::ADDWrs:
1870     Is64Bit = false;
1871     return AArch64::ADDSWrs;
1872   case AArch64::ADDWrx:
1873     Is64Bit = false;
1874     return AArch64::ADDSWrx;
1875   case AArch64::ANDWri:
1876     Is64Bit = false;
1877     return AArch64::ANDSWri;
1878   case AArch64::ANDWrr:
1879     Is64Bit = false;
1880     return AArch64::ANDSWrr;
1881   case AArch64::ANDWrs:
1882     Is64Bit = false;
1883     return AArch64::ANDSWrs;
1884   case AArch64::BICWrr:
1885     Is64Bit = false;
1886     return AArch64::BICSWrr;
1887   case AArch64::BICWrs:
1888     Is64Bit = false;
1889     return AArch64::BICSWrs;
1890   case AArch64::SUBWri:
1891     Is64Bit = false;
1892     return AArch64::SUBSWri;
1893   case AArch64::SUBWrr:
1894     Is64Bit = false;
1895     return AArch64::SUBSWrr;
1896   case AArch64::SUBWrs:
1897     Is64Bit = false;
1898     return AArch64::SUBSWrs;
1899   case AArch64::SUBWrx:
1900     Is64Bit = false;
1901     return AArch64::SUBSWrx;
1902   // 64-bit cases:
1903   case AArch64::ADDXri:
1904     Is64Bit = true;
1905     return AArch64::ADDSXri;
1906   case AArch64::ADDXrr:
1907     Is64Bit = true;
1908     return AArch64::ADDSXrr;
1909   case AArch64::ADDXrs:
1910     Is64Bit = true;
1911     return AArch64::ADDSXrs;
1912   case AArch64::ADDXrx:
1913     Is64Bit = true;
1914     return AArch64::ADDSXrx;
1915   case AArch64::ANDXri:
1916     Is64Bit = true;
1917     return AArch64::ANDSXri;
1918   case AArch64::ANDXrr:
1919     Is64Bit = true;
1920     return AArch64::ANDSXrr;
1921   case AArch64::ANDXrs:
1922     Is64Bit = true;
1923     return AArch64::ANDSXrs;
1924   case AArch64::BICXrr:
1925     Is64Bit = true;
1926     return AArch64::BICSXrr;
1927   case AArch64::BICXrs:
1928     Is64Bit = true;
1929     return AArch64::BICSXrs;
1930   case AArch64::SUBXri:
1931     Is64Bit = true;
1932     return AArch64::SUBSXri;
1933   case AArch64::SUBXrr:
1934     Is64Bit = true;
1935     return AArch64::SUBSXrr;
1936   case AArch64::SUBXrs:
1937     Is64Bit = true;
1938     return AArch64::SUBSXrs;
1939   case AArch64::SUBXrx:
1940     Is64Bit = true;
1941     return AArch64::SUBSXrx;
1942   }
1943 }
1944 
1945 // Is this a candidate for ld/st merging or pairing?  For example, we don't
1946 // touch volatiles or load/stores that have a hint to avoid pair formation.
1947 bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
1948   // If this is a volatile load/store, don't mess with it.
1949   if (MI.hasOrderedMemoryRef())
1950     return false;
1951 
1952   // Make sure this is a reg/fi+imm (as opposed to an address reloc).
1953   assert((MI.getOperand(1).isReg() || MI.getOperand(1).isFI()) &&
1954          "Expected a reg or frame index operand.");
1955   if (!MI.getOperand(2).isImm())
1956     return false;
1957 
1958   // Can't merge/pair if the instruction modifies the base register.
1959   // e.g., ldr x0, [x0]
1960   // This case will never occur with an FI base.
1961   if (MI.getOperand(1).isReg()) {
1962     Register BaseReg = MI.getOperand(1).getReg();
1963     const TargetRegisterInfo *TRI = &getRegisterInfo();
1964     if (MI.modifiesRegister(BaseReg, TRI))
1965       return false;
1966   }
1967 
1968   // Check if this load/store has a hint to avoid pair formation.
1969   // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
1970   if (isLdStPairSuppressed(MI))
1971     return false;
1972 
1973   // Do not pair any callee-save store/reload instructions in the
1974   // prologue/epilogue if the CFI information encoded the operations as separate
1975   // instructions, as that will cause the size of the actual prologue to mismatch
1976   // with the prologue size recorded in the Windows CFI.
1977   const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo();
1978   bool NeedsWinCFI = MAI->usesWindowsCFI() &&
1979                      MI.getMF()->getFunction().needsUnwindTableEntry();
1980   if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
1981                       MI.getFlag(MachineInstr::FrameDestroy)))
1982     return false;
1983 
1984   // On some CPUs quad load/store pairs are slower than two single load/stores.
1985   if (Subtarget.isPaired128Slow()) {
1986     switch (MI.getOpcode()) {
1987     default:
1988       break;
1989     case AArch64::LDURQi:
1990     case AArch64::STURQi:
1991     case AArch64::LDRQui:
1992     case AArch64::STRQui:
1993       return false;
1994     }
1995   }
1996 
1997   return true;
1998 }
1999 
2000 bool AArch64InstrInfo::getMemOperandsWithOffset(
2001     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2002     int64_t &Offset, const TargetRegisterInfo *TRI) const {
2003   if (!LdSt.mayLoadOrStore())
2004     return false;
2005 
2006   const MachineOperand *BaseOp;
2007   unsigned Width;
2008   if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2009     return false;
2010   BaseOps.push_back(BaseOp);
2011   return true;
2012 }
2013 
2014 bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
2015     const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
2016     unsigned &Width, const TargetRegisterInfo *TRI) const {
2017   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2018   // Handle only loads/stores with base register followed by immediate offset.
2019   if (LdSt.getNumExplicitOperands() == 3) {
2020     // Non-paired instruction (e.g., ldr x1, [x0, #8]).
2021     if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2022         !LdSt.getOperand(2).isImm())
2023       return false;
2024   } else if (LdSt.getNumExplicitOperands() == 4) {
2025     // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
2026     if (!LdSt.getOperand(1).isReg() ||
2027         (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
2028         !LdSt.getOperand(3).isImm())
2029       return false;
2030   } else
2031     return false;
2032 
2033   // Get the scaling factor for the instruction and set the width for the
2034   // instruction.
2035   unsigned Scale = 0;
2036   int64_t Dummy1, Dummy2;
2037 
2038   // If this returns false, then it's an instruction we don't want to handle.
2039   if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
2040     return false;
2041 
2042   // Compute the offset. Offset is calculated as the immediate operand
2043   // multiplied by the scaling factor. Unscaled instructions have scaling factor
2044   // set to 1.
2045   if (LdSt.getNumExplicitOperands() == 3) {
2046     BaseOp = &LdSt.getOperand(1);
2047     Offset = LdSt.getOperand(2).getImm() * Scale;
2048   } else {
2049     assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
2050     BaseOp = &LdSt.getOperand(2);
2051     Offset = LdSt.getOperand(3).getImm() * Scale;
2052   }
2053 
2054   if (!BaseOp->isReg() && !BaseOp->isFI())
2055     return false;
2056 
2057   return true;
2058 }
2059 
2060 MachineOperand &
2061 AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
2062   assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2063   MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
2064   assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
2065   return OfsOp;
2066 }
2067 
2068 bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
2069                                     unsigned &Width, int64_t &MinOffset,
2070                                     int64_t &MaxOffset) {
2071   const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
2072   switch (Opcode) {
2073   // Not a memory operation or something we want to handle.
2074   default:
2075     Scale = Width = 0;
2076     MinOffset = MaxOffset = 0;
2077     return false;
2078   case AArch64::STRWpost:
2079   case AArch64::LDRWpost:
2080     Width = 32;
2081     Scale = 4;
2082     MinOffset = -256;
2083     MaxOffset = 255;
2084     break;
2085   case AArch64::LDURQi:
2086   case AArch64::STURQi:
2087     Width = 16;
2088     Scale = 1;
2089     MinOffset = -256;
2090     MaxOffset = 255;
2091     break;
2092   case AArch64::PRFUMi:
2093   case AArch64::LDURXi:
2094   case AArch64::LDURDi:
2095   case AArch64::STURXi:
2096   case AArch64::STURDi:
2097     Width = 8;
2098     Scale = 1;
2099     MinOffset = -256;
2100     MaxOffset = 255;
2101     break;
2102   case AArch64::LDURWi:
2103   case AArch64::LDURSi:
2104   case AArch64::LDURSWi:
2105   case AArch64::STURWi:
2106   case AArch64::STURSi:
2107     Width = 4;
2108     Scale = 1;
2109     MinOffset = -256;
2110     MaxOffset = 255;
2111     break;
2112   case AArch64::LDURHi:
2113   case AArch64::LDURHHi:
2114   case AArch64::LDURSHXi:
2115   case AArch64::LDURSHWi:
2116   case AArch64::STURHi:
2117   case AArch64::STURHHi:
2118     Width = 2;
2119     Scale = 1;
2120     MinOffset = -256;
2121     MaxOffset = 255;
2122     break;
2123   case AArch64::LDURBi:
2124   case AArch64::LDURBBi:
2125   case AArch64::LDURSBXi:
2126   case AArch64::LDURSBWi:
2127   case AArch64::STURBi:
2128   case AArch64::STURBBi:
2129     Width = 1;
2130     Scale = 1;
2131     MinOffset = -256;
2132     MaxOffset = 255;
2133     break;
2134   case AArch64::LDPQi:
2135   case AArch64::LDNPQi:
2136   case AArch64::STPQi:
2137   case AArch64::STNPQi:
2138     Scale = 16;
2139     Width = 32;
2140     MinOffset = -64;
2141     MaxOffset = 63;
2142     break;
2143   case AArch64::LDRQui:
2144   case AArch64::STRQui:
2145     Scale = Width = 16;
2146     MinOffset = 0;
2147     MaxOffset = 4095;
2148     break;
2149   case AArch64::LDPXi:
2150   case AArch64::LDPDi:
2151   case AArch64::LDNPXi:
2152   case AArch64::LDNPDi:
2153   case AArch64::STPXi:
2154   case AArch64::STPDi:
2155   case AArch64::STNPXi:
2156   case AArch64::STNPDi:
2157     Scale = 8;
2158     Width = 16;
2159     MinOffset = -64;
2160     MaxOffset = 63;
2161     break;
2162   case AArch64::PRFMui:
2163   case AArch64::LDRXui:
2164   case AArch64::LDRDui:
2165   case AArch64::STRXui:
2166   case AArch64::STRDui:
2167     Scale = Width = 8;
2168     MinOffset = 0;
2169     MaxOffset = 4095;
2170     break;
2171   case AArch64::LDPWi:
2172   case AArch64::LDPSi:
2173   case AArch64::LDNPWi:
2174   case AArch64::LDNPSi:
2175   case AArch64::STPWi:
2176   case AArch64::STPSi:
2177   case AArch64::STNPWi:
2178   case AArch64::STNPSi:
2179     Scale = 4;
2180     Width = 8;
2181     MinOffset = -64;
2182     MaxOffset = 63;
2183     break;
2184   case AArch64::LDRWui:
2185   case AArch64::LDRSui:
2186   case AArch64::LDRSWui:
2187   case AArch64::STRWui:
2188   case AArch64::STRSui:
2189     Scale = Width = 4;
2190     MinOffset = 0;
2191     MaxOffset = 4095;
2192     break;
2193   case AArch64::LDRHui:
2194   case AArch64::LDRHHui:
2195   case AArch64::LDRSHWui:
2196   case AArch64::LDRSHXui:
2197   case AArch64::STRHui:
2198   case AArch64::STRHHui:
2199     Scale = Width = 2;
2200     MinOffset = 0;
2201     MaxOffset = 4095;
2202     break;
2203   case AArch64::LDRBui:
2204   case AArch64::LDRBBui:
2205   case AArch64::LDRSBWui:
2206   case AArch64::LDRSBXui:
2207   case AArch64::STRBui:
2208   case AArch64::STRBBui:
2209     Scale = Width = 1;
2210     MinOffset = 0;
2211     MaxOffset = 4095;
2212     break;
2213   case AArch64::ADDG:
2214     Scale = 16;
2215     Width = 0;
2216     MinOffset = 0;
2217     MaxOffset = 63;
2218     break;
2219   case AArch64::TAGPstack:
2220     Scale = 16;
2221     Width = 0;
2222     // TAGP with a negative offset turns into SUBP, which has a maximum offset
2223     // of 63 (not 64!).
2224     MinOffset = -63;
2225     MaxOffset = 63;
2226     break;
2227   case AArch64::LDG:
2228   case AArch64::STGOffset:
2229   case AArch64::STZGOffset:
2230     Scale = Width = 16;
2231     MinOffset = -256;
2232     MaxOffset = 255;
2233     break;
2234   case AArch64::LDR_PXI:
2235   case AArch64::STR_PXI:
2236     Scale = 2;
2237     Width = SVEMaxBytesPerVector / 8;
2238     MinOffset = -256;
2239     MaxOffset = 255;
2240     break;
2241   case AArch64::LDR_ZXI:
2242   case AArch64::STR_ZXI:
2243     Scale = 16;
2244     Width = SVEMaxBytesPerVector;
2245     MinOffset = -256;
2246     MaxOffset = 255;
2247     break;
2248   case AArch64::LD1B_IMM:
2249   case AArch64::LD1H_IMM:
2250   case AArch64::LD1W_IMM:
2251   case AArch64::LD1D_IMM:
2252   case AArch64::ST1B_IMM:
2253   case AArch64::ST1H_IMM:
2254   case AArch64::ST1W_IMM:
2255   case AArch64::ST1D_IMM:
2256     // A full vectors worth of data
2257     // Width = mbytes * elements
2258     Scale = 16;
2259     Width = SVEMaxBytesPerVector;
2260     MinOffset = -8;
2261     MaxOffset = 7;
2262     break;
2263   case AArch64::ST2GOffset:
2264   case AArch64::STZ2GOffset:
2265     Scale = 16;
2266     Width = 32;
2267     MinOffset = -256;
2268     MaxOffset = 255;
2269     break;
2270   case AArch64::STGPi:
2271     Scale = Width = 16;
2272     MinOffset = -64;
2273     MaxOffset = 63;
2274     break;
2275   }
2276 
2277   return true;
2278 }
2279 
2280 // Scaling factor for unscaled load or store.
2281 int AArch64InstrInfo::getMemScale(unsigned Opc) {
2282   switch (Opc) {
2283   default:
2284     llvm_unreachable("Opcode has unknown scale!");
2285   case AArch64::LDRBBui:
2286   case AArch64::LDURBBi:
2287   case AArch64::LDRSBWui:
2288   case AArch64::LDURSBWi:
2289   case AArch64::STRBBui:
2290   case AArch64::STURBBi:
2291     return 1;
2292   case AArch64::LDRHHui:
2293   case AArch64::LDURHHi:
2294   case AArch64::LDRSHWui:
2295   case AArch64::LDURSHWi:
2296   case AArch64::STRHHui:
2297   case AArch64::STURHHi:
2298     return 2;
2299   case AArch64::LDRSui:
2300   case AArch64::LDURSi:
2301   case AArch64::LDRSWui:
2302   case AArch64::LDURSWi:
2303   case AArch64::LDRWui:
2304   case AArch64::LDURWi:
2305   case AArch64::STRSui:
2306   case AArch64::STURSi:
2307   case AArch64::STRWui:
2308   case AArch64::STURWi:
2309   case AArch64::LDPSi:
2310   case AArch64::LDPSWi:
2311   case AArch64::LDPWi:
2312   case AArch64::STPSi:
2313   case AArch64::STPWi:
2314     return 4;
2315   case AArch64::LDRDui:
2316   case AArch64::LDURDi:
2317   case AArch64::LDRXui:
2318   case AArch64::LDURXi:
2319   case AArch64::STRDui:
2320   case AArch64::STURDi:
2321   case AArch64::STRXui:
2322   case AArch64::STURXi:
2323   case AArch64::LDPDi:
2324   case AArch64::LDPXi:
2325   case AArch64::STPDi:
2326   case AArch64::STPXi:
2327     return 8;
2328   case AArch64::LDRQui:
2329   case AArch64::LDURQi:
2330   case AArch64::STRQui:
2331   case AArch64::STURQi:
2332   case AArch64::LDPQi:
2333   case AArch64::STPQi:
2334   case AArch64::STGOffset:
2335   case AArch64::STZGOffset:
2336   case AArch64::ST2GOffset:
2337   case AArch64::STZ2GOffset:
2338   case AArch64::STGPi:
2339     return 16;
2340   }
2341 }
2342 
2343 // Scale the unscaled offsets.  Returns false if the unscaled offset can't be
2344 // scaled.
2345 static bool scaleOffset(unsigned Opc, int64_t &Offset) {
2346   int Scale = AArch64InstrInfo::getMemScale(Opc);
2347 
2348   // If the byte-offset isn't a multiple of the stride, we can't scale this
2349   // offset.
2350   if (Offset % Scale != 0)
2351     return false;
2352 
2353   // Convert the byte-offset used by unscaled into an "element" offset used
2354   // by the scaled pair load/store instructions.
2355   Offset /= Scale;
2356   return true;
2357 }
2358 
2359 static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
2360   if (FirstOpc == SecondOpc)
2361     return true;
2362   // We can also pair sign-ext and zero-ext instructions.
2363   switch (FirstOpc) {
2364   default:
2365     return false;
2366   case AArch64::LDRWui:
2367   case AArch64::LDURWi:
2368     return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
2369   case AArch64::LDRSWui:
2370   case AArch64::LDURSWi:
2371     return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
2372   }
2373   // These instructions can't be paired based on their opcodes.
2374   return false;
2375 }
2376 
2377 static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
2378                             int64_t Offset1, unsigned Opcode1, int FI2,
2379                             int64_t Offset2, unsigned Opcode2) {
2380   // Accesses through fixed stack object frame indices may access a different
2381   // fixed stack slot. Check that the object offsets + offsets match.
2382   if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
2383     int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
2384     int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
2385     assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
2386     // Convert to scaled object offsets.
2387     int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
2388     if (ObjectOffset1 % Scale1 != 0)
2389       return false;
2390     ObjectOffset1 /= Scale1;
2391     int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
2392     if (ObjectOffset2 % Scale2 != 0)
2393       return false;
2394     ObjectOffset2 /= Scale2;
2395     ObjectOffset1 += Offset1;
2396     ObjectOffset2 += Offset2;
2397     return ObjectOffset1 + 1 == ObjectOffset2;
2398   }
2399 
2400   return FI1 == FI2;
2401 }
2402 
2403 /// Detect opportunities for ldp/stp formation.
2404 ///
2405 /// Only called for LdSt for which getMemOperandWithOffset returns true.
2406 bool AArch64InstrInfo::shouldClusterMemOps(
2407     ArrayRef<const MachineOperand *> BaseOps1,
2408     ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads) const {
2409   assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2410   const MachineOperand &BaseOp1 = *BaseOps1.front();
2411   const MachineOperand &BaseOp2 = *BaseOps2.front();
2412   const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2413   const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2414   if (BaseOp1.getType() != BaseOp2.getType())
2415     return false;
2416 
2417   assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2418          "Only base registers and frame indices are supported.");
2419 
2420   // Check for both base regs and base FI.
2421   if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
2422     return false;
2423 
2424   // Only cluster up to a single pair.
2425   if (NumLoads > 2)
2426     return false;
2427 
2428   if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
2429     return false;
2430 
2431   // Can we pair these instructions based on their opcodes?
2432   unsigned FirstOpc = FirstLdSt.getOpcode();
2433   unsigned SecondOpc = SecondLdSt.getOpcode();
2434   if (!canPairLdStOpc(FirstOpc, SecondOpc))
2435     return false;
2436 
2437   // Can't merge volatiles or load/stores that have a hint to avoid pair
2438   // formation, for example.
2439   if (!isCandidateToMergeOrPair(FirstLdSt) ||
2440       !isCandidateToMergeOrPair(SecondLdSt))
2441     return false;
2442 
2443   // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
2444   int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
2445   if (isUnscaledLdSt(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
2446     return false;
2447 
2448   int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
2449   if (isUnscaledLdSt(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
2450     return false;
2451 
2452   // Pairwise instructions have a 7-bit signed offset field.
2453   if (Offset1 > 63 || Offset1 < -64)
2454     return false;
2455 
2456   // The caller should already have ordered First/SecondLdSt by offset.
2457   // Note: except for non-equal frame index bases
2458   if (BaseOp1.isFI()) {
2459     assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
2460            "Caller should have ordered offsets.");
2461 
2462     const MachineFrameInfo &MFI =
2463         FirstLdSt.getParent()->getParent()->getFrameInfo();
2464     return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
2465                            BaseOp2.getIndex(), Offset2, SecondOpc);
2466   }
2467 
2468   assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2469 
2470   return Offset1 + 1 == Offset2;
2471 }
2472 
2473 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
2474                                             unsigned Reg, unsigned SubIdx,
2475                                             unsigned State,
2476                                             const TargetRegisterInfo *TRI) {
2477   if (!SubIdx)
2478     return MIB.addReg(Reg, State);
2479 
2480   if (Register::isPhysicalRegister(Reg))
2481     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
2482   return MIB.addReg(Reg, State, SubIdx);
2483 }
2484 
2485 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
2486                                         unsigned NumRegs) {
2487   // We really want the positive remainder mod 32 here, that happens to be
2488   // easily obtainable with a mask.
2489   return ((DestReg - SrcReg) & 0x1f) < NumRegs;
2490 }
2491 
2492 void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
2493                                         MachineBasicBlock::iterator I,
2494                                         const DebugLoc &DL, MCRegister DestReg,
2495                                         MCRegister SrcReg, bool KillSrc,
2496                                         unsigned Opcode,
2497                                         ArrayRef<unsigned> Indices) const {
2498   assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
2499   const TargetRegisterInfo *TRI = &getRegisterInfo();
2500   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2501   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2502   unsigned NumRegs = Indices.size();
2503 
2504   int SubReg = 0, End = NumRegs, Incr = 1;
2505   if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
2506     SubReg = NumRegs - 1;
2507     End = -1;
2508     Incr = -1;
2509   }
2510 
2511   for (; SubReg != End; SubReg += Incr) {
2512     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2513     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2514     AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
2515     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2516   }
2517 }
2518 
2519 void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
2520                                        MachineBasicBlock::iterator I,
2521                                        DebugLoc DL, unsigned DestReg,
2522                                        unsigned SrcReg, bool KillSrc,
2523                                        unsigned Opcode, unsigned ZeroReg,
2524                                        llvm::ArrayRef<unsigned> Indices) const {
2525   const TargetRegisterInfo *TRI = &getRegisterInfo();
2526   unsigned NumRegs = Indices.size();
2527 
2528 #ifndef NDEBUG
2529   uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
2530   uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
2531   assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
2532          "GPR reg sequences should not be able to overlap");
2533 #endif
2534 
2535   for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
2536     const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
2537     AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
2538     MIB.addReg(ZeroReg);
2539     AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
2540     MIB.addImm(0);
2541   }
2542 }
2543 
2544 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
2545                                    MachineBasicBlock::iterator I,
2546                                    const DebugLoc &DL, MCRegister DestReg,
2547                                    MCRegister SrcReg, bool KillSrc) const {
2548   if (AArch64::GPR32spRegClass.contains(DestReg) &&
2549       (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
2550     const TargetRegisterInfo *TRI = &getRegisterInfo();
2551 
2552     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
2553       // If either operand is WSP, expand to ADD #0.
2554       if (Subtarget.hasZeroCycleRegMove()) {
2555         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
2556         MCRegister DestRegX = TRI->getMatchingSuperReg(
2557             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2558         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2559             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2560         // This instruction is reading and writing X registers.  This may upset
2561         // the register scavenger and machine verifier, so we need to indicate
2562         // that we are reading an undefined value from SrcRegX, but a proper
2563         // value from SrcReg.
2564         BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
2565             .addReg(SrcRegX, RegState::Undef)
2566             .addImm(0)
2567             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2568             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2569       } else {
2570         BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
2571             .addReg(SrcReg, getKillRegState(KillSrc))
2572             .addImm(0)
2573             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2574       }
2575     } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
2576       BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
2577           .addImm(0)
2578           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2579     } else {
2580       if (Subtarget.hasZeroCycleRegMove()) {
2581         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
2582         MCRegister DestRegX = TRI->getMatchingSuperReg(
2583             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2584         MCRegister SrcRegX = TRI->getMatchingSuperReg(
2585             SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
2586         // This instruction is reading and writing X registers.  This may upset
2587         // the register scavenger and machine verifier, so we need to indicate
2588         // that we are reading an undefined value from SrcRegX, but a proper
2589         // value from SrcReg.
2590         BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
2591             .addReg(AArch64::XZR)
2592             .addReg(SrcRegX, RegState::Undef)
2593             .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
2594       } else {
2595         // Otherwise, expand to ORR WZR.
2596         BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
2597             .addReg(AArch64::WZR)
2598             .addReg(SrcReg, getKillRegState(KillSrc));
2599       }
2600     }
2601     return;
2602   }
2603 
2604   // Copy a Predicate register by ORRing with itself.
2605   if (AArch64::PPRRegClass.contains(DestReg) &&
2606       AArch64::PPRRegClass.contains(SrcReg)) {
2607     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2608     BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
2609       .addReg(SrcReg) // Pg
2610       .addReg(SrcReg)
2611       .addReg(SrcReg, getKillRegState(KillSrc));
2612     return;
2613   }
2614 
2615   // Copy a Z register by ORRing with itself.
2616   if (AArch64::ZPRRegClass.contains(DestReg) &&
2617       AArch64::ZPRRegClass.contains(SrcReg)) {
2618     assert(Subtarget.hasSVE() && "Unexpected SVE register.");
2619     BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
2620       .addReg(SrcReg)
2621       .addReg(SrcReg, getKillRegState(KillSrc));
2622     return;
2623   }
2624 
2625   if (AArch64::GPR64spRegClass.contains(DestReg) &&
2626       (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
2627     if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
2628       // If either operand is SP, expand to ADD #0.
2629       BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
2630           .addReg(SrcReg, getKillRegState(KillSrc))
2631           .addImm(0)
2632           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2633     } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
2634       BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
2635           .addImm(0)
2636           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
2637     } else {
2638       // Otherwise, expand to ORR XZR.
2639       BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
2640           .addReg(AArch64::XZR)
2641           .addReg(SrcReg, getKillRegState(KillSrc));
2642     }
2643     return;
2644   }
2645 
2646   // Copy a DDDD register quad by copying the individual sub-registers.
2647   if (AArch64::DDDDRegClass.contains(DestReg) &&
2648       AArch64::DDDDRegClass.contains(SrcReg)) {
2649     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2650                                        AArch64::dsub2, AArch64::dsub3};
2651     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2652                      Indices);
2653     return;
2654   }
2655 
2656   // Copy a DDD register triple by copying the individual sub-registers.
2657   if (AArch64::DDDRegClass.contains(DestReg) &&
2658       AArch64::DDDRegClass.contains(SrcReg)) {
2659     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
2660                                        AArch64::dsub2};
2661     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2662                      Indices);
2663     return;
2664   }
2665 
2666   // Copy a DD register pair by copying the individual sub-registers.
2667   if (AArch64::DDRegClass.contains(DestReg) &&
2668       AArch64::DDRegClass.contains(SrcReg)) {
2669     static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
2670     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
2671                      Indices);
2672     return;
2673   }
2674 
2675   // Copy a QQQQ register quad by copying the individual sub-registers.
2676   if (AArch64::QQQQRegClass.contains(DestReg) &&
2677       AArch64::QQQQRegClass.contains(SrcReg)) {
2678     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2679                                        AArch64::qsub2, AArch64::qsub3};
2680     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2681                      Indices);
2682     return;
2683   }
2684 
2685   // Copy a QQQ register triple by copying the individual sub-registers.
2686   if (AArch64::QQQRegClass.contains(DestReg) &&
2687       AArch64::QQQRegClass.contains(SrcReg)) {
2688     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
2689                                        AArch64::qsub2};
2690     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2691                      Indices);
2692     return;
2693   }
2694 
2695   // Copy a QQ register pair by copying the individual sub-registers.
2696   if (AArch64::QQRegClass.contains(DestReg) &&
2697       AArch64::QQRegClass.contains(SrcReg)) {
2698     static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
2699     copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
2700                      Indices);
2701     return;
2702   }
2703 
2704   if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
2705       AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
2706     static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
2707     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
2708                     AArch64::XZR, Indices);
2709     return;
2710   }
2711 
2712   if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
2713       AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
2714     static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
2715     copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
2716                     AArch64::WZR, Indices);
2717     return;
2718   }
2719 
2720   if (AArch64::FPR128RegClass.contains(DestReg) &&
2721       AArch64::FPR128RegClass.contains(SrcReg)) {
2722     if (Subtarget.hasNEON()) {
2723       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2724           .addReg(SrcReg)
2725           .addReg(SrcReg, getKillRegState(KillSrc));
2726     } else {
2727       BuildMI(MBB, I, DL, get(AArch64::STRQpre))
2728           .addReg(AArch64::SP, RegState::Define)
2729           .addReg(SrcReg, getKillRegState(KillSrc))
2730           .addReg(AArch64::SP)
2731           .addImm(-16);
2732       BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
2733           .addReg(AArch64::SP, RegState::Define)
2734           .addReg(DestReg, RegState::Define)
2735           .addReg(AArch64::SP)
2736           .addImm(16);
2737     }
2738     return;
2739   }
2740 
2741   if (AArch64::FPR64RegClass.contains(DestReg) &&
2742       AArch64::FPR64RegClass.contains(SrcReg)) {
2743     if (Subtarget.hasNEON()) {
2744       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
2745                                        &AArch64::FPR128RegClass);
2746       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
2747                                       &AArch64::FPR128RegClass);
2748       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2749           .addReg(SrcReg)
2750           .addReg(SrcReg, getKillRegState(KillSrc));
2751     } else {
2752       BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
2753           .addReg(SrcReg, getKillRegState(KillSrc));
2754     }
2755     return;
2756   }
2757 
2758   if (AArch64::FPR32RegClass.contains(DestReg) &&
2759       AArch64::FPR32RegClass.contains(SrcReg)) {
2760     if (Subtarget.hasNEON()) {
2761       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
2762                                        &AArch64::FPR128RegClass);
2763       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
2764                                       &AArch64::FPR128RegClass);
2765       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2766           .addReg(SrcReg)
2767           .addReg(SrcReg, getKillRegState(KillSrc));
2768     } else {
2769       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2770           .addReg(SrcReg, getKillRegState(KillSrc));
2771     }
2772     return;
2773   }
2774 
2775   if (AArch64::FPR16RegClass.contains(DestReg) &&
2776       AArch64::FPR16RegClass.contains(SrcReg)) {
2777     if (Subtarget.hasNEON()) {
2778       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2779                                        &AArch64::FPR128RegClass);
2780       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2781                                       &AArch64::FPR128RegClass);
2782       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2783           .addReg(SrcReg)
2784           .addReg(SrcReg, getKillRegState(KillSrc));
2785     } else {
2786       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
2787                                        &AArch64::FPR32RegClass);
2788       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
2789                                       &AArch64::FPR32RegClass);
2790       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2791           .addReg(SrcReg, getKillRegState(KillSrc));
2792     }
2793     return;
2794   }
2795 
2796   if (AArch64::FPR8RegClass.contains(DestReg) &&
2797       AArch64::FPR8RegClass.contains(SrcReg)) {
2798     if (Subtarget.hasNEON()) {
2799       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2800                                        &AArch64::FPR128RegClass);
2801       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2802                                       &AArch64::FPR128RegClass);
2803       BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
2804           .addReg(SrcReg)
2805           .addReg(SrcReg, getKillRegState(KillSrc));
2806     } else {
2807       DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
2808                                        &AArch64::FPR32RegClass);
2809       SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
2810                                       &AArch64::FPR32RegClass);
2811       BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
2812           .addReg(SrcReg, getKillRegState(KillSrc));
2813     }
2814     return;
2815   }
2816 
2817   // Copies between GPR64 and FPR64.
2818   if (AArch64::FPR64RegClass.contains(DestReg) &&
2819       AArch64::GPR64RegClass.contains(SrcReg)) {
2820     BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
2821         .addReg(SrcReg, getKillRegState(KillSrc));
2822     return;
2823   }
2824   if (AArch64::GPR64RegClass.contains(DestReg) &&
2825       AArch64::FPR64RegClass.contains(SrcReg)) {
2826     BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
2827         .addReg(SrcReg, getKillRegState(KillSrc));
2828     return;
2829   }
2830   // Copies between GPR32 and FPR32.
2831   if (AArch64::FPR32RegClass.contains(DestReg) &&
2832       AArch64::GPR32RegClass.contains(SrcReg)) {
2833     BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
2834         .addReg(SrcReg, getKillRegState(KillSrc));
2835     return;
2836   }
2837   if (AArch64::GPR32RegClass.contains(DestReg) &&
2838       AArch64::FPR32RegClass.contains(SrcReg)) {
2839     BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
2840         .addReg(SrcReg, getKillRegState(KillSrc));
2841     return;
2842   }
2843 
2844   if (DestReg == AArch64::NZCV) {
2845     assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
2846     BuildMI(MBB, I, DL, get(AArch64::MSR))
2847         .addImm(AArch64SysReg::NZCV)
2848         .addReg(SrcReg, getKillRegState(KillSrc))
2849         .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
2850     return;
2851   }
2852 
2853   if (SrcReg == AArch64::NZCV) {
2854     assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
2855     BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
2856         .addImm(AArch64SysReg::NZCV)
2857         .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
2858     return;
2859   }
2860 
2861   llvm_unreachable("unimplemented reg-to-reg copy");
2862 }
2863 
2864 static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
2865                                     MachineBasicBlock &MBB,
2866                                     MachineBasicBlock::iterator InsertBefore,
2867                                     const MCInstrDesc &MCID,
2868                                     Register SrcReg, bool IsKill,
2869                                     unsigned SubIdx0, unsigned SubIdx1, int FI,
2870                                     MachineMemOperand *MMO) {
2871   Register SrcReg0 = SrcReg;
2872   Register SrcReg1 = SrcReg;
2873   if (Register::isPhysicalRegister(SrcReg)) {
2874     SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
2875     SubIdx0 = 0;
2876     SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
2877     SubIdx1 = 0;
2878   }
2879   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
2880       .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
2881       .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
2882       .addFrameIndex(FI)
2883       .addImm(0)
2884       .addMemOperand(MMO);
2885 }
2886 
2887 void AArch64InstrInfo::storeRegToStackSlot(
2888     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
2889     bool isKill, int FI, const TargetRegisterClass *RC,
2890     const TargetRegisterInfo *TRI) const {
2891   MachineFunction &MF = *MBB.getParent();
2892   MachineFrameInfo &MFI = MF.getFrameInfo();
2893   unsigned Align = MFI.getObjectAlignment(FI);
2894 
2895   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
2896   MachineMemOperand *MMO = MF.getMachineMemOperand(
2897       PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
2898   unsigned Opc = 0;
2899   bool Offset = true;
2900   switch (TRI->getSpillSize(*RC)) {
2901   case 1:
2902     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
2903       Opc = AArch64::STRBui;
2904     break;
2905   case 2:
2906     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
2907       Opc = AArch64::STRHui;
2908     break;
2909   case 4:
2910     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
2911       Opc = AArch64::STRWui;
2912       if (Register::isVirtualRegister(SrcReg))
2913         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
2914       else
2915         assert(SrcReg != AArch64::WSP);
2916     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
2917       Opc = AArch64::STRSui;
2918     break;
2919   case 8:
2920     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
2921       Opc = AArch64::STRXui;
2922       if (Register::isVirtualRegister(SrcReg))
2923         MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
2924       else
2925         assert(SrcReg != AArch64::SP);
2926     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
2927       Opc = AArch64::STRDui;
2928     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
2929       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
2930                               get(AArch64::STPWi), SrcReg, isKill,
2931                               AArch64::sube32, AArch64::subo32, FI, MMO);
2932       return;
2933     }
2934     break;
2935   case 16:
2936     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
2937       Opc = AArch64::STRQui;
2938     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
2939       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2940       Opc = AArch64::ST1Twov1d;
2941       Offset = false;
2942     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
2943       storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
2944                               get(AArch64::STPXi), SrcReg, isKill,
2945                               AArch64::sube64, AArch64::subo64, FI, MMO);
2946       return;
2947     }
2948     break;
2949   case 24:
2950     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
2951       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2952       Opc = AArch64::ST1Threev1d;
2953       Offset = false;
2954     }
2955     break;
2956   case 32:
2957     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
2958       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2959       Opc = AArch64::ST1Fourv1d;
2960       Offset = false;
2961     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
2962       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2963       Opc = AArch64::ST1Twov2d;
2964       Offset = false;
2965     }
2966     break;
2967   case 48:
2968     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
2969       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2970       Opc = AArch64::ST1Threev2d;
2971       Offset = false;
2972     }
2973     break;
2974   case 64:
2975     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
2976       assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
2977       Opc = AArch64::ST1Fourv2d;
2978       Offset = false;
2979     }
2980     break;
2981   }
2982   unsigned StackID = TargetStackID::Default;
2983   if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
2984     assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
2985     Opc = AArch64::STR_PXI;
2986     StackID = TargetStackID::SVEVector;
2987   } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
2988     assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
2989     Opc = AArch64::STR_ZXI;
2990     StackID = TargetStackID::SVEVector;
2991   }
2992   assert(Opc && "Unknown register class");
2993   MFI.setStackID(FI, StackID);
2994 
2995   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
2996                                      .addReg(SrcReg, getKillRegState(isKill))
2997                                      .addFrameIndex(FI);
2998 
2999   if (Offset)
3000     MI.addImm(0);
3001   MI.addMemOperand(MMO);
3002 }
3003 
3004 static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
3005                                      MachineBasicBlock &MBB,
3006                                      MachineBasicBlock::iterator InsertBefore,
3007                                      const MCInstrDesc &MCID,
3008                                      Register DestReg, unsigned SubIdx0,
3009                                      unsigned SubIdx1, int FI,
3010                                      MachineMemOperand *MMO) {
3011   Register DestReg0 = DestReg;
3012   Register DestReg1 = DestReg;
3013   bool IsUndef = true;
3014   if (Register::isPhysicalRegister(DestReg)) {
3015     DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
3016     SubIdx0 = 0;
3017     DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
3018     SubIdx1 = 0;
3019     IsUndef = false;
3020   }
3021   BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3022       .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
3023       .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
3024       .addFrameIndex(FI)
3025       .addImm(0)
3026       .addMemOperand(MMO);
3027 }
3028 
3029 void AArch64InstrInfo::loadRegFromStackSlot(
3030     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
3031     int FI, const TargetRegisterClass *RC,
3032     const TargetRegisterInfo *TRI) const {
3033   MachineFunction &MF = *MBB.getParent();
3034   MachineFrameInfo &MFI = MF.getFrameInfo();
3035   unsigned Align = MFI.getObjectAlignment(FI);
3036   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
3037   MachineMemOperand *MMO = MF.getMachineMemOperand(
3038       PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
3039 
3040   unsigned Opc = 0;
3041   bool Offset = true;
3042   switch (TRI->getSpillSize(*RC)) {
3043   case 1:
3044     if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3045       Opc = AArch64::LDRBui;
3046     break;
3047   case 2:
3048     if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3049       Opc = AArch64::LDRHui;
3050     break;
3051   case 4:
3052     if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3053       Opc = AArch64::LDRWui;
3054       if (Register::isVirtualRegister(DestReg))
3055         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
3056       else
3057         assert(DestReg != AArch64::WSP);
3058     } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3059       Opc = AArch64::LDRSui;
3060     break;
3061   case 8:
3062     if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3063       Opc = AArch64::LDRXui;
3064       if (Register::isVirtualRegister(DestReg))
3065         MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
3066       else
3067         assert(DestReg != AArch64::SP);
3068     } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3069       Opc = AArch64::LDRDui;
3070     } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3071       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3072                                get(AArch64::LDPWi), DestReg, AArch64::sube32,
3073                                AArch64::subo32, FI, MMO);
3074       return;
3075     }
3076     break;
3077   case 16:
3078     if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3079       Opc = AArch64::LDRQui;
3080     else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3081       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3082       Opc = AArch64::LD1Twov1d;
3083       Offset = false;
3084     } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3085       loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
3086                                get(AArch64::LDPXi), DestReg, AArch64::sube64,
3087                                AArch64::subo64, FI, MMO);
3088       return;
3089     }
3090     break;
3091   case 24:
3092     if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3093       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3094       Opc = AArch64::LD1Threev1d;
3095       Offset = false;
3096     }
3097     break;
3098   case 32:
3099     if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3100       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3101       Opc = AArch64::LD1Fourv1d;
3102       Offset = false;
3103     } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3104       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3105       Opc = AArch64::LD1Twov2d;
3106       Offset = false;
3107     }
3108     break;
3109   case 48:
3110     if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3111       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3112       Opc = AArch64::LD1Threev2d;
3113       Offset = false;
3114     }
3115     break;
3116   case 64:
3117     if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3118       assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
3119       Opc = AArch64::LD1Fourv2d;
3120       Offset = false;
3121     }
3122     break;
3123   }
3124 
3125   unsigned StackID = TargetStackID::Default;
3126   if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3127     assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3128     Opc = AArch64::LDR_PXI;
3129     StackID = TargetStackID::SVEVector;
3130   } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3131     assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
3132     Opc = AArch64::LDR_ZXI;
3133     StackID = TargetStackID::SVEVector;
3134   }
3135   assert(Opc && "Unknown register class");
3136   MFI.setStackID(FI, StackID);
3137 
3138   const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3139                                      .addReg(DestReg, getDefRegState(true))
3140                                      .addFrameIndex(FI);
3141   if (Offset)
3142     MI.addImm(0);
3143   MI.addMemOperand(MMO);
3144 }
3145 
3146 // Helper function to emit a frame offset adjustment from a given
3147 // pointer (SrcReg), stored into DestReg. This function is explicit
3148 // in that it requires the opcode.
3149 static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
3150                                MachineBasicBlock::iterator MBBI,
3151                                const DebugLoc &DL, unsigned DestReg,
3152                                unsigned SrcReg, int64_t Offset, unsigned Opc,
3153                                const TargetInstrInfo *TII,
3154                                MachineInstr::MIFlag Flag, bool NeedsWinCFI,
3155                                bool *HasWinCFI) {
3156   int Sign = 1;
3157   unsigned MaxEncoding, ShiftSize;
3158   switch (Opc) {
3159   case AArch64::ADDXri:
3160   case AArch64::ADDSXri:
3161   case AArch64::SUBXri:
3162   case AArch64::SUBSXri:
3163     MaxEncoding = 0xfff;
3164     ShiftSize = 12;
3165     break;
3166   case AArch64::ADDVL_XXI:
3167   case AArch64::ADDPL_XXI:
3168     MaxEncoding = 31;
3169     ShiftSize = 0;
3170     if (Offset < 0) {
3171       MaxEncoding = 32;
3172       Sign = -1;
3173       Offset = -Offset;
3174     }
3175     break;
3176   default:
3177     llvm_unreachable("Unsupported opcode");
3178   }
3179 
3180   // FIXME: If the offset won't fit in 24-bits, compute the offset into a
3181   // scratch register.  If DestReg is a virtual register, use it as the
3182   // scratch register; otherwise, create a new virtual register (to be
3183   // replaced by the scavenger at the end of PEI).  That case can be optimized
3184   // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
3185   // register can be loaded with offset%8 and the add/sub can use an extending
3186   // instruction with LSL#3.
3187   // Currently the function handles any offsets but generates a poor sequence
3188   // of code.
3189   //  assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
3190 
3191   const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
3192   do {
3193     uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
3194     unsigned LocalShiftSize = 0;
3195     if (ThisVal > MaxEncoding) {
3196       ThisVal = ThisVal >> ShiftSize;
3197       LocalShiftSize = ShiftSize;
3198     }
3199     assert((ThisVal >> ShiftSize) <= MaxEncoding &&
3200            "Encoding cannot handle value that big");
3201     auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
3202                    .addReg(SrcReg)
3203                    .addImm(Sign * (int)ThisVal);
3204     if (ShiftSize)
3205       MBI = MBI.addImm(
3206           AArch64_AM::getShifterImm(AArch64_AM::LSL, LocalShiftSize));
3207     MBI = MBI.setMIFlag(Flag);
3208 
3209     if (NeedsWinCFI) {
3210       assert(Sign == 1 && "SEH directives should always have a positive sign");
3211       int Imm = (int)(ThisVal << LocalShiftSize);
3212       if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
3213           (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
3214         if (HasWinCFI)
3215           *HasWinCFI = true;
3216         if (Imm == 0)
3217           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
3218         else
3219           BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
3220               .addImm(Imm)
3221               .setMIFlag(Flag);
3222         assert((Offset - Imm) == 0 && "Expected remaining offset to be zero to "
3223                                       "emit a single SEH directive");
3224       } else if (DestReg == AArch64::SP) {
3225         if (HasWinCFI)
3226           *HasWinCFI = true;
3227         assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
3228         BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
3229             .addImm(Imm)
3230             .setMIFlag(Flag);
3231       }
3232       if (HasWinCFI)
3233         *HasWinCFI = true;
3234     }
3235 
3236     SrcReg = DestReg;
3237     Offset -= ThisVal << LocalShiftSize;
3238   } while (Offset);
3239 }
3240 
3241 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
3242                            MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
3243                            unsigned DestReg, unsigned SrcReg,
3244                            StackOffset Offset, const TargetInstrInfo *TII,
3245                            MachineInstr::MIFlag Flag, bool SetNZCV,
3246                            bool NeedsWinCFI, bool *HasWinCFI) {
3247   int64_t Bytes, NumPredicateVectors, NumDataVectors;
3248   Offset.getForFrameOffset(Bytes, NumPredicateVectors, NumDataVectors);
3249 
3250   // First emit non-scalable frame offsets, or a simple 'mov'.
3251   if (Bytes || (!Offset && SrcReg != DestReg)) {
3252     assert((DestReg != AArch64::SP || Bytes % 16 == 0) &&
3253            "SP increment/decrement not 16-byte aligned");
3254     unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
3255     if (Bytes < 0) {
3256       Bytes = -Bytes;
3257       Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
3258     }
3259     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
3260                        NeedsWinCFI, HasWinCFI);
3261     SrcReg = DestReg;
3262   }
3263 
3264   assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
3265          "SetNZCV not supported with SVE vectors");
3266   assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
3267          "WinCFI not supported with SVE vectors");
3268 
3269   if (NumDataVectors) {
3270     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
3271                        AArch64::ADDVL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3272     SrcReg = DestReg;
3273   }
3274 
3275   if (NumPredicateVectors) {
3276     assert(DestReg != AArch64::SP && "Unaligned access to SP");
3277     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
3278                        AArch64::ADDPL_XXI, TII, Flag, NeedsWinCFI, nullptr);
3279   }
3280 }
3281 
3282 MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
3283     MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
3284     MachineBasicBlock::iterator InsertPt, int FrameIndex,
3285     LiveIntervals *LIS, VirtRegMap *VRM) const {
3286   // This is a bit of a hack. Consider this instruction:
3287   //
3288   //   %0 = COPY %sp; GPR64all:%0
3289   //
3290   // We explicitly chose GPR64all for the virtual register so such a copy might
3291   // be eliminated by RegisterCoalescer. However, that may not be possible, and
3292   // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
3293   // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
3294   //
3295   // To prevent that, we are going to constrain the %0 register class here.
3296   //
3297   // <rdar://problem/11522048>
3298   //
3299   if (MI.isFullCopy()) {
3300     Register DstReg = MI.getOperand(0).getReg();
3301     Register SrcReg = MI.getOperand(1).getReg();
3302     if (SrcReg == AArch64::SP && Register::isVirtualRegister(DstReg)) {
3303       MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
3304       return nullptr;
3305     }
3306     if (DstReg == AArch64::SP && Register::isVirtualRegister(SrcReg)) {
3307       MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3308       return nullptr;
3309     }
3310   }
3311 
3312   // Handle the case where a copy is being spilled or filled but the source
3313   // and destination register class don't match.  For example:
3314   //
3315   //   %0 = COPY %xzr; GPR64common:%0
3316   //
3317   // In this case we can still safely fold away the COPY and generate the
3318   // following spill code:
3319   //
3320   //   STRXui %xzr, %stack.0
3321   //
3322   // This also eliminates spilled cross register class COPYs (e.g. between x and
3323   // d regs) of the same size.  For example:
3324   //
3325   //   %0 = COPY %1; GPR64:%0, FPR64:%1
3326   //
3327   // will be filled as
3328   //
3329   //   LDRDui %0, fi<#0>
3330   //
3331   // instead of
3332   //
3333   //   LDRXui %Temp, fi<#0>
3334   //   %0 = FMOV %Temp
3335   //
3336   if (MI.isCopy() && Ops.size() == 1 &&
3337       // Make sure we're only folding the explicit COPY defs/uses.
3338       (Ops[0] == 0 || Ops[0] == 1)) {
3339     bool IsSpill = Ops[0] == 0;
3340     bool IsFill = !IsSpill;
3341     const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
3342     const MachineRegisterInfo &MRI = MF.getRegInfo();
3343     MachineBasicBlock &MBB = *MI.getParent();
3344     const MachineOperand &DstMO = MI.getOperand(0);
3345     const MachineOperand &SrcMO = MI.getOperand(1);
3346     Register DstReg = DstMO.getReg();
3347     Register SrcReg = SrcMO.getReg();
3348     // This is slightly expensive to compute for physical regs since
3349     // getMinimalPhysRegClass is slow.
3350     auto getRegClass = [&](unsigned Reg) {
3351       return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
3352                                               : TRI.getMinimalPhysRegClass(Reg);
3353     };
3354 
3355     if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
3356       assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
3357                  TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
3358              "Mismatched register size in non subreg COPY");
3359       if (IsSpill)
3360         storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
3361                             getRegClass(SrcReg), &TRI);
3362       else
3363         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
3364                              getRegClass(DstReg), &TRI);
3365       return &*--InsertPt;
3366     }
3367 
3368     // Handle cases like spilling def of:
3369     //
3370     //   %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
3371     //
3372     // where the physical register source can be widened and stored to the full
3373     // virtual reg destination stack slot, in this case producing:
3374     //
3375     //   STRXui %xzr, %stack.0
3376     //
3377     if (IsSpill && DstMO.isUndef() && Register::isPhysicalRegister(SrcReg)) {
3378       assert(SrcMO.getSubReg() == 0 &&
3379              "Unexpected subreg on physical register");
3380       const TargetRegisterClass *SpillRC;
3381       unsigned SpillSubreg;
3382       switch (DstMO.getSubReg()) {
3383       default:
3384         SpillRC = nullptr;
3385         break;
3386       case AArch64::sub_32:
3387       case AArch64::ssub:
3388         if (AArch64::GPR32RegClass.contains(SrcReg)) {
3389           SpillRC = &AArch64::GPR64RegClass;
3390           SpillSubreg = AArch64::sub_32;
3391         } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
3392           SpillRC = &AArch64::FPR64RegClass;
3393           SpillSubreg = AArch64::ssub;
3394         } else
3395           SpillRC = nullptr;
3396         break;
3397       case AArch64::dsub:
3398         if (AArch64::FPR64RegClass.contains(SrcReg)) {
3399           SpillRC = &AArch64::FPR128RegClass;
3400           SpillSubreg = AArch64::dsub;
3401         } else
3402           SpillRC = nullptr;
3403         break;
3404       }
3405 
3406       if (SpillRC)
3407         if (unsigned WidenedSrcReg =
3408                 TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
3409           storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
3410                               FrameIndex, SpillRC, &TRI);
3411           return &*--InsertPt;
3412         }
3413     }
3414 
3415     // Handle cases like filling use of:
3416     //
3417     //   %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
3418     //
3419     // where we can load the full virtual reg source stack slot, into the subreg
3420     // destination, in this case producing:
3421     //
3422     //   LDRWui %0:sub_32<def,read-undef>, %stack.0
3423     //
3424     if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
3425       const TargetRegisterClass *FillRC;
3426       switch (DstMO.getSubReg()) {
3427       default:
3428         FillRC = nullptr;
3429         break;
3430       case AArch64::sub_32:
3431         FillRC = &AArch64::GPR32RegClass;
3432         break;
3433       case AArch64::ssub:
3434         FillRC = &AArch64::FPR32RegClass;
3435         break;
3436       case AArch64::dsub:
3437         FillRC = &AArch64::FPR64RegClass;
3438         break;
3439       }
3440 
3441       if (FillRC) {
3442         assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
3443                    TRI.getRegSizeInBits(*FillRC) &&
3444                "Mismatched regclass size on folded subreg COPY");
3445         loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC, &TRI);
3446         MachineInstr &LoadMI = *--InsertPt;
3447         MachineOperand &LoadDst = LoadMI.getOperand(0);
3448         assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
3449         LoadDst.setSubReg(DstMO.getSubReg());
3450         LoadDst.setIsUndef();
3451         return &LoadMI;
3452       }
3453     }
3454   }
3455 
3456   // Cannot fold.
3457   return nullptr;
3458 }
3459 
3460 static bool isSVEScaledImmInstruction(unsigned Opcode) {
3461   switch (Opcode) {
3462   case AArch64::LDR_ZXI:
3463   case AArch64::STR_ZXI:
3464   case AArch64::LDR_PXI:
3465   case AArch64::STR_PXI:
3466   case AArch64::LD1B_IMM:
3467   case AArch64::LD1H_IMM:
3468   case AArch64::LD1W_IMM:
3469   case AArch64::LD1D_IMM:
3470   case AArch64::ST1B_IMM:
3471   case AArch64::ST1H_IMM:
3472   case AArch64::ST1W_IMM:
3473   case AArch64::ST1D_IMM:
3474     return true;
3475   default:
3476     return false;
3477   }
3478 }
3479 
3480 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
3481                                     StackOffset &SOffset,
3482                                     bool *OutUseUnscaledOp,
3483                                     unsigned *OutUnscaledOp,
3484                                     int64_t *EmittableOffset) {
3485   // Set output values in case of early exit.
3486   if (EmittableOffset)
3487     *EmittableOffset = 0;
3488   if (OutUseUnscaledOp)
3489     *OutUseUnscaledOp = false;
3490   if (OutUnscaledOp)
3491     *OutUnscaledOp = 0;
3492 
3493   // Exit early for structured vector spills/fills as they can't take an
3494   // immediate offset.
3495   switch (MI.getOpcode()) {
3496   default:
3497     break;
3498   case AArch64::LD1Twov2d:
3499   case AArch64::LD1Threev2d:
3500   case AArch64::LD1Fourv2d:
3501   case AArch64::LD1Twov1d:
3502   case AArch64::LD1Threev1d:
3503   case AArch64::LD1Fourv1d:
3504   case AArch64::ST1Twov2d:
3505   case AArch64::ST1Threev2d:
3506   case AArch64::ST1Fourv2d:
3507   case AArch64::ST1Twov1d:
3508   case AArch64::ST1Threev1d:
3509   case AArch64::ST1Fourv1d:
3510   case AArch64::IRG:
3511   case AArch64::IRGstack:
3512   case AArch64::STGloop:
3513   case AArch64::STZGloop:
3514     return AArch64FrameOffsetCannotUpdate;
3515   }
3516 
3517   // Get the min/max offset and the scale.
3518   unsigned Scale, Width;
3519   int64_t MinOff, MaxOff;
3520   if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), Scale, Width, MinOff,
3521                                       MaxOff))
3522     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3523 
3524   // Construct the complete offset.
3525   bool IsMulVL = isSVEScaledImmInstruction(MI.getOpcode());
3526   int64_t Offset =
3527       IsMulVL ? (SOffset.getScalableBytes()) : (SOffset.getBytes());
3528 
3529   const MachineOperand &ImmOpnd =
3530       MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
3531   Offset += ImmOpnd.getImm() * Scale;
3532 
3533   // If the offset doesn't match the scale, we rewrite the instruction to
3534   // use the unscaled instruction instead. Likewise, if we have a negative
3535   // offset and there is an unscaled op to use.
3536   Optional<unsigned> UnscaledOp =
3537       AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
3538   bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
3539   if (useUnscaledOp &&
3540       !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, Scale, Width, MinOff, MaxOff))
3541     llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
3542 
3543   int64_t Remainder = Offset % Scale;
3544   assert(!(Remainder && useUnscaledOp) &&
3545          "Cannot have remainder when using unscaled op");
3546 
3547   assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
3548   int64_t NewOffset = Offset / Scale;
3549   if (MinOff <= NewOffset && NewOffset <= MaxOff)
3550     Offset = Remainder;
3551   else {
3552     NewOffset = NewOffset < 0 ? MinOff : MaxOff;
3553     Offset = Offset - NewOffset * Scale + Remainder;
3554   }
3555 
3556   if (EmittableOffset)
3557     *EmittableOffset = NewOffset;
3558   if (OutUseUnscaledOp)
3559     *OutUseUnscaledOp = useUnscaledOp;
3560   if (OutUnscaledOp && UnscaledOp)
3561     *OutUnscaledOp = *UnscaledOp;
3562 
3563   if (IsMulVL)
3564     SOffset = StackOffset(Offset, MVT::nxv1i8) +
3565               StackOffset(SOffset.getBytes(), MVT::i8);
3566   else
3567     SOffset = StackOffset(Offset, MVT::i8) +
3568               StackOffset(SOffset.getScalableBytes(), MVT::nxv1i8);
3569   return AArch64FrameOffsetCanUpdate |
3570          (SOffset ? 0 : AArch64FrameOffsetIsLegal);
3571 }
3572 
3573 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
3574                                     unsigned FrameReg, StackOffset &Offset,
3575                                     const AArch64InstrInfo *TII) {
3576   unsigned Opcode = MI.getOpcode();
3577   unsigned ImmIdx = FrameRegIdx + 1;
3578 
3579   if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
3580     Offset += StackOffset(MI.getOperand(ImmIdx).getImm(), MVT::i8);
3581     emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
3582                     MI.getOperand(0).getReg(), FrameReg, Offset, TII,
3583                     MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
3584     MI.eraseFromParent();
3585     Offset = StackOffset();
3586     return true;
3587   }
3588 
3589   int64_t NewOffset;
3590   unsigned UnscaledOp;
3591   bool UseUnscaledOp;
3592   int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
3593                                          &UnscaledOp, &NewOffset);
3594   if (Status & AArch64FrameOffsetCanUpdate) {
3595     if (Status & AArch64FrameOffsetIsLegal)
3596       // Replace the FrameIndex with FrameReg.
3597       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
3598     if (UseUnscaledOp)
3599       MI.setDesc(TII->get(UnscaledOp));
3600 
3601     MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
3602     return !Offset;
3603   }
3604 
3605   return false;
3606 }
3607 
3608 void AArch64InstrInfo::getNoop(MCInst &NopInst) const {
3609   NopInst.setOpcode(AArch64::HINT);
3610   NopInst.addOperand(MCOperand::createImm(0));
3611 }
3612 
3613 // AArch64 supports MachineCombiner.
3614 bool AArch64InstrInfo::useMachineCombiner() const { return true; }
3615 
3616 // True when Opc sets flag
3617 static bool isCombineInstrSettingFlag(unsigned Opc) {
3618   switch (Opc) {
3619   case AArch64::ADDSWrr:
3620   case AArch64::ADDSWri:
3621   case AArch64::ADDSXrr:
3622   case AArch64::ADDSXri:
3623   case AArch64::SUBSWrr:
3624   case AArch64::SUBSXrr:
3625   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3626   case AArch64::SUBSWri:
3627   case AArch64::SUBSXri:
3628     return true;
3629   default:
3630     break;
3631   }
3632   return false;
3633 }
3634 
3635 // 32b Opcodes that can be combined with a MUL
3636 static bool isCombineInstrCandidate32(unsigned Opc) {
3637   switch (Opc) {
3638   case AArch64::ADDWrr:
3639   case AArch64::ADDWri:
3640   case AArch64::SUBWrr:
3641   case AArch64::ADDSWrr:
3642   case AArch64::ADDSWri:
3643   case AArch64::SUBSWrr:
3644   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3645   case AArch64::SUBWri:
3646   case AArch64::SUBSWri:
3647     return true;
3648   default:
3649     break;
3650   }
3651   return false;
3652 }
3653 
3654 // 64b Opcodes that can be combined with a MUL
3655 static bool isCombineInstrCandidate64(unsigned Opc) {
3656   switch (Opc) {
3657   case AArch64::ADDXrr:
3658   case AArch64::ADDXri:
3659   case AArch64::SUBXrr:
3660   case AArch64::ADDSXrr:
3661   case AArch64::ADDSXri:
3662   case AArch64::SUBSXrr:
3663   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
3664   case AArch64::SUBXri:
3665   case AArch64::SUBSXri:
3666   case AArch64::ADDv8i8:
3667   case AArch64::ADDv16i8:
3668   case AArch64::ADDv4i16:
3669   case AArch64::ADDv8i16:
3670   case AArch64::ADDv2i32:
3671   case AArch64::ADDv4i32:
3672   case AArch64::SUBv8i8:
3673   case AArch64::SUBv16i8:
3674   case AArch64::SUBv4i16:
3675   case AArch64::SUBv8i16:
3676   case AArch64::SUBv2i32:
3677   case AArch64::SUBv4i32:
3678     return true;
3679   default:
3680     break;
3681   }
3682   return false;
3683 }
3684 
3685 // FP Opcodes that can be combined with a FMUL
3686 static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
3687   switch (Inst.getOpcode()) {
3688   default:
3689     break;
3690   case AArch64::FADDHrr:
3691   case AArch64::FADDSrr:
3692   case AArch64::FADDDrr:
3693   case AArch64::FADDv4f16:
3694   case AArch64::FADDv8f16:
3695   case AArch64::FADDv2f32:
3696   case AArch64::FADDv2f64:
3697   case AArch64::FADDv4f32:
3698   case AArch64::FSUBHrr:
3699   case AArch64::FSUBSrr:
3700   case AArch64::FSUBDrr:
3701   case AArch64::FSUBv4f16:
3702   case AArch64::FSUBv8f16:
3703   case AArch64::FSUBv2f32:
3704   case AArch64::FSUBv2f64:
3705   case AArch64::FSUBv4f32:
3706     TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
3707     return (Options.UnsafeFPMath ||
3708             Options.AllowFPOpFusion == FPOpFusion::Fast);
3709   }
3710   return false;
3711 }
3712 
3713 // Opcodes that can be combined with a MUL
3714 static bool isCombineInstrCandidate(unsigned Opc) {
3715   return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
3716 }
3717 
3718 //
3719 // Utility routine that checks if \param MO is defined by an
3720 // \param CombineOpc instruction in the basic block \param MBB
3721 static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
3722                        unsigned CombineOpc, unsigned ZeroReg = 0,
3723                        bool CheckZeroReg = false) {
3724   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3725   MachineInstr *MI = nullptr;
3726 
3727   if (MO.isReg() && Register::isVirtualRegister(MO.getReg()))
3728     MI = MRI.getUniqueVRegDef(MO.getReg());
3729   // And it needs to be in the trace (otherwise, it won't have a depth).
3730   if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc)
3731     return false;
3732   // Must only used by the user we combine with.
3733   if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
3734     return false;
3735 
3736   if (CheckZeroReg) {
3737     assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
3738            MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
3739            MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
3740     // The third input reg must be zero.
3741     if (MI->getOperand(3).getReg() != ZeroReg)
3742       return false;
3743   }
3744 
3745   return true;
3746 }
3747 
3748 //
3749 // Is \param MO defined by an integer multiply and can be combined?
3750 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3751                               unsigned MulOpc, unsigned ZeroReg) {
3752   return canCombine(MBB, MO, MulOpc, ZeroReg, true);
3753 }
3754 
3755 //
3756 // Is \param MO defined by a floating-point multiply and can be combined?
3757 static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
3758                                unsigned MulOpc) {
3759   return canCombine(MBB, MO, MulOpc);
3760 }
3761 
3762 // TODO: There are many more machine instruction opcodes to match:
3763 //       1. Other data types (integer, vectors)
3764 //       2. Other math / logic operations (xor, or)
3765 //       3. Other forms of the same operation (intrinsics and other variants)
3766 bool AArch64InstrInfo::isAssociativeAndCommutative(
3767     const MachineInstr &Inst) const {
3768   switch (Inst.getOpcode()) {
3769   case AArch64::FADDDrr:
3770   case AArch64::FADDSrr:
3771   case AArch64::FADDv2f32:
3772   case AArch64::FADDv2f64:
3773   case AArch64::FADDv4f32:
3774   case AArch64::FMULDrr:
3775   case AArch64::FMULSrr:
3776   case AArch64::FMULX32:
3777   case AArch64::FMULX64:
3778   case AArch64::FMULXv2f32:
3779   case AArch64::FMULXv2f64:
3780   case AArch64::FMULXv4f32:
3781   case AArch64::FMULv2f32:
3782   case AArch64::FMULv2f64:
3783   case AArch64::FMULv4f32:
3784     return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath;
3785   default:
3786     return false;
3787   }
3788 }
3789 
3790 /// Find instructions that can be turned into madd.
3791 static bool getMaddPatterns(MachineInstr &Root,
3792                             SmallVectorImpl<MachineCombinerPattern> &Patterns) {
3793   unsigned Opc = Root.getOpcode();
3794   MachineBasicBlock &MBB = *Root.getParent();
3795   bool Found = false;
3796 
3797   if (!isCombineInstrCandidate(Opc))
3798     return false;
3799   if (isCombineInstrSettingFlag(Opc)) {
3800     int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
3801     // When NZCV is live bail out.
3802     if (Cmp_NZCV == -1)
3803       return false;
3804     unsigned NewOpc = convertToNonFlagSettingOpc(Root);
3805     // When opcode can't change bail out.
3806     // CHECKME: do we miss any cases for opcode conversion?
3807     if (NewOpc == Opc)
3808       return false;
3809     Opc = NewOpc;
3810   }
3811 
3812   auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
3813                       MachineCombinerPattern Pattern) {
3814     if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
3815       Patterns.push_back(Pattern);
3816       Found = true;
3817     }
3818   };
3819 
3820   auto setVFound = [&](int Opcode, int Operand, MachineCombinerPattern Pattern) {
3821     if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
3822       Patterns.push_back(Pattern);
3823       Found = true;
3824     }
3825   };
3826 
3827   typedef MachineCombinerPattern MCP;
3828 
3829   switch (Opc) {
3830   default:
3831     break;
3832   case AArch64::ADDWrr:
3833     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
3834            "ADDWrr does not have register operands");
3835     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
3836     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
3837     break;
3838   case AArch64::ADDXrr:
3839     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
3840     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
3841     break;
3842   case AArch64::SUBWrr:
3843     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
3844     setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
3845     break;
3846   case AArch64::SUBXrr:
3847     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
3848     setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
3849     break;
3850   case AArch64::ADDWri:
3851     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
3852     break;
3853   case AArch64::ADDXri:
3854     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
3855     break;
3856   case AArch64::SUBWri:
3857     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
3858     break;
3859   case AArch64::SUBXri:
3860     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
3861     break;
3862   case AArch64::ADDv8i8:
3863     setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
3864     setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
3865     break;
3866   case AArch64::ADDv16i8:
3867     setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
3868     setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
3869     break;
3870   case AArch64::ADDv4i16:
3871     setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
3872     setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
3873     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
3874     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
3875     break;
3876   case AArch64::ADDv8i16:
3877     setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
3878     setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
3879     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
3880     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
3881     break;
3882   case AArch64::ADDv2i32:
3883     setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
3884     setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
3885     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
3886     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
3887     break;
3888   case AArch64::ADDv4i32:
3889     setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
3890     setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
3891     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
3892     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
3893     break;
3894   case AArch64::SUBv8i8:
3895     setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
3896     setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
3897     break;
3898   case AArch64::SUBv16i8:
3899     setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
3900     setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
3901     break;
3902   case AArch64::SUBv4i16:
3903     setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
3904     setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
3905     setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
3906     setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
3907     break;
3908   case AArch64::SUBv8i16:
3909     setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
3910     setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
3911     setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
3912     setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
3913     break;
3914   case AArch64::SUBv2i32:
3915     setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
3916     setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
3917     setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
3918     setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
3919     break;
3920   case AArch64::SUBv4i32:
3921     setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
3922     setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
3923     setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
3924     setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
3925     break;
3926   }
3927   return Found;
3928 }
3929 /// Floating-Point Support
3930 
3931 /// Find instructions that can be turned into madd.
3932 static bool getFMAPatterns(MachineInstr &Root,
3933                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {
3934 
3935   if (!isCombineInstrCandidateFP(Root))
3936     return false;
3937 
3938   MachineBasicBlock &MBB = *Root.getParent();
3939   bool Found = false;
3940 
3941   auto Match = [&](int Opcode, int Operand,
3942                    MachineCombinerPattern Pattern) -> bool {
3943     if (canCombineWithFMUL(MBB, Root.getOperand(Operand), Opcode)) {
3944       Patterns.push_back(Pattern);
3945       return true;
3946     }
3947     return false;
3948   };
3949 
3950   typedef MachineCombinerPattern MCP;
3951 
3952   switch (Root.getOpcode()) {
3953   default:
3954     assert(false && "Unsupported FP instruction in combiner\n");
3955     break;
3956   case AArch64::FADDHrr:
3957     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
3958            "FADDHrr does not have register operands");
3959 
3960     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
3961     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
3962     break;
3963   case AArch64::FADDSrr:
3964     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
3965            "FADDSrr does not have register operands");
3966 
3967     Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
3968              Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
3969 
3970     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
3971              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
3972     break;
3973   case AArch64::FADDDrr:
3974     Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
3975              Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
3976 
3977     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
3978              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
3979     break;
3980   case AArch64::FADDv4f16:
3981     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
3982              Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
3983 
3984     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
3985              Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
3986     break;
3987   case AArch64::FADDv8f16:
3988     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
3989              Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
3990 
3991     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
3992              Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
3993     break;
3994   case AArch64::FADDv2f32:
3995     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
3996              Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
3997 
3998     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
3999              Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
4000     break;
4001   case AArch64::FADDv2f64:
4002     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
4003              Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
4004 
4005     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
4006              Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
4007     break;
4008   case AArch64::FADDv4f32:
4009     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
4010              Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
4011 
4012     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
4013              Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
4014     break;
4015   case AArch64::FSUBHrr:
4016     Found  = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
4017     Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
4018     Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
4019     break;
4020   case AArch64::FSUBSrr:
4021     Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
4022 
4023     Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
4024              Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
4025 
4026     Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
4027     break;
4028   case AArch64::FSUBDrr:
4029     Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
4030 
4031     Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
4032              Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
4033 
4034     Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
4035     break;
4036   case AArch64::FSUBv4f16:
4037     Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
4038              Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
4039 
4040     Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
4041              Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
4042     break;
4043   case AArch64::FSUBv8f16:
4044     Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
4045              Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
4046 
4047     Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
4048              Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
4049     break;
4050   case AArch64::FSUBv2f32:
4051     Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
4052              Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
4053 
4054     Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
4055              Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
4056     break;
4057   case AArch64::FSUBv2f64:
4058     Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
4059              Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
4060 
4061     Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
4062              Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
4063     break;
4064   case AArch64::FSUBv4f32:
4065     Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
4066              Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
4067 
4068     Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
4069              Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
4070     break;
4071   }
4072   return Found;
4073 }
4074 
4075 /// Return true when a code sequence can improve throughput. It
4076 /// should be called only for instructions in loops.
4077 /// \param Pattern - combiner pattern
4078 bool AArch64InstrInfo::isThroughputPattern(
4079     MachineCombinerPattern Pattern) const {
4080   switch (Pattern) {
4081   default:
4082     break;
4083   case MachineCombinerPattern::FMULADDH_OP1:
4084   case MachineCombinerPattern::FMULADDH_OP2:
4085   case MachineCombinerPattern::FMULSUBH_OP1:
4086   case MachineCombinerPattern::FMULSUBH_OP2:
4087   case MachineCombinerPattern::FMULADDS_OP1:
4088   case MachineCombinerPattern::FMULADDS_OP2:
4089   case MachineCombinerPattern::FMULSUBS_OP1:
4090   case MachineCombinerPattern::FMULSUBS_OP2:
4091   case MachineCombinerPattern::FMULADDD_OP1:
4092   case MachineCombinerPattern::FMULADDD_OP2:
4093   case MachineCombinerPattern::FMULSUBD_OP1:
4094   case MachineCombinerPattern::FMULSUBD_OP2:
4095   case MachineCombinerPattern::FNMULSUBH_OP1:
4096   case MachineCombinerPattern::FNMULSUBS_OP1:
4097   case MachineCombinerPattern::FNMULSUBD_OP1:
4098   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
4099   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
4100   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
4101   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
4102   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4103   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4104   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
4105   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
4106   case MachineCombinerPattern::FMLAv4f16_OP2:
4107   case MachineCombinerPattern::FMLAv4f16_OP1:
4108   case MachineCombinerPattern::FMLAv8f16_OP1:
4109   case MachineCombinerPattern::FMLAv8f16_OP2:
4110   case MachineCombinerPattern::FMLAv2f32_OP2:
4111   case MachineCombinerPattern::FMLAv2f32_OP1:
4112   case MachineCombinerPattern::FMLAv2f64_OP1:
4113   case MachineCombinerPattern::FMLAv2f64_OP2:
4114   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
4115   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
4116   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
4117   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
4118   case MachineCombinerPattern::FMLAv4f32_OP1:
4119   case MachineCombinerPattern::FMLAv4f32_OP2:
4120   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
4121   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
4122   case MachineCombinerPattern::FMLSv4i16_indexed_OP1:
4123   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
4124   case MachineCombinerPattern::FMLSv8i16_indexed_OP1:
4125   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
4126   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
4127   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
4128   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
4129   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
4130   case MachineCombinerPattern::FMLSv4f16_OP1:
4131   case MachineCombinerPattern::FMLSv4f16_OP2:
4132   case MachineCombinerPattern::FMLSv8f16_OP1:
4133   case MachineCombinerPattern::FMLSv8f16_OP2:
4134   case MachineCombinerPattern::FMLSv2f32_OP2:
4135   case MachineCombinerPattern::FMLSv2f64_OP2:
4136   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
4137   case MachineCombinerPattern::FMLSv4f32_OP2:
4138   case MachineCombinerPattern::MULADDv8i8_OP1:
4139   case MachineCombinerPattern::MULADDv8i8_OP2:
4140   case MachineCombinerPattern::MULADDv16i8_OP1:
4141   case MachineCombinerPattern::MULADDv16i8_OP2:
4142   case MachineCombinerPattern::MULADDv4i16_OP1:
4143   case MachineCombinerPattern::MULADDv4i16_OP2:
4144   case MachineCombinerPattern::MULADDv8i16_OP1:
4145   case MachineCombinerPattern::MULADDv8i16_OP2:
4146   case MachineCombinerPattern::MULADDv2i32_OP1:
4147   case MachineCombinerPattern::MULADDv2i32_OP2:
4148   case MachineCombinerPattern::MULADDv4i32_OP1:
4149   case MachineCombinerPattern::MULADDv4i32_OP2:
4150   case MachineCombinerPattern::MULSUBv8i8_OP1:
4151   case MachineCombinerPattern::MULSUBv8i8_OP2:
4152   case MachineCombinerPattern::MULSUBv16i8_OP1:
4153   case MachineCombinerPattern::MULSUBv16i8_OP2:
4154   case MachineCombinerPattern::MULSUBv4i16_OP1:
4155   case MachineCombinerPattern::MULSUBv4i16_OP2:
4156   case MachineCombinerPattern::MULSUBv8i16_OP1:
4157   case MachineCombinerPattern::MULSUBv8i16_OP2:
4158   case MachineCombinerPattern::MULSUBv2i32_OP1:
4159   case MachineCombinerPattern::MULSUBv2i32_OP2:
4160   case MachineCombinerPattern::MULSUBv4i32_OP1:
4161   case MachineCombinerPattern::MULSUBv4i32_OP2:
4162   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4163   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4164   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4165   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4166   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4167   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4168   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4169   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4170   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4171   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4172   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4173   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4174   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4175   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4176   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4177   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4178     return true;
4179   } // end switch (Pattern)
4180   return false;
4181 }
4182 /// Return true when there is potentially a faster code sequence for an
4183 /// instruction chain ending in \p Root. All potential patterns are listed in
4184 /// the \p Pattern vector. Pattern should be sorted in priority order since the
4185 /// pattern evaluator stops checking as soon as it finds a faster sequence.
4186 
4187 bool AArch64InstrInfo::getMachineCombinerPatterns(
4188     MachineInstr &Root,
4189     SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
4190   // Integer patterns
4191   if (getMaddPatterns(Root, Patterns))
4192     return true;
4193   // Floating point patterns
4194   if (getFMAPatterns(Root, Patterns))
4195     return true;
4196 
4197   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
4198 }
4199 
4200 enum class FMAInstKind { Default, Indexed, Accumulator };
4201 /// genFusedMultiply - Generate fused multiply instructions.
4202 /// This function supports both integer and floating point instructions.
4203 /// A typical example:
4204 ///  F|MUL I=A,B,0
4205 ///  F|ADD R,I,C
4206 ///  ==> F|MADD R,A,B,C
4207 /// \param MF Containing MachineFunction
4208 /// \param MRI Register information
4209 /// \param TII Target information
4210 /// \param Root is the F|ADD instruction
4211 /// \param [out] InsInstrs is a vector of machine instructions and will
4212 /// contain the generated madd instruction
4213 /// \param IdxMulOpd is index of operand in Root that is the result of
4214 /// the F|MUL. In the example above IdxMulOpd is 1.
4215 /// \param MaddOpc the opcode fo the f|madd instruction
4216 /// \param RC Register class of operands
4217 /// \param kind of fma instruction (addressing mode) to be generated
4218 /// \param ReplacedAddend is the result register from the instruction
4219 /// replacing the non-combined operand, if any.
4220 static MachineInstr *
4221 genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
4222                  const TargetInstrInfo *TII, MachineInstr &Root,
4223                  SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
4224                  unsigned MaddOpc, const TargetRegisterClass *RC,
4225                  FMAInstKind kind = FMAInstKind::Default,
4226                  const Register *ReplacedAddend = nullptr) {
4227   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4228 
4229   unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
4230   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4231   Register ResultReg = Root.getOperand(0).getReg();
4232   Register SrcReg0 = MUL->getOperand(1).getReg();
4233   bool Src0IsKill = MUL->getOperand(1).isKill();
4234   Register SrcReg1 = MUL->getOperand(2).getReg();
4235   bool Src1IsKill = MUL->getOperand(2).isKill();
4236 
4237   unsigned SrcReg2;
4238   bool Src2IsKill;
4239   if (ReplacedAddend) {
4240     // If we just generated a new addend, we must be it's only use.
4241     SrcReg2 = *ReplacedAddend;
4242     Src2IsKill = true;
4243   } else {
4244     SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
4245     Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
4246   }
4247 
4248   if (Register::isVirtualRegister(ResultReg))
4249     MRI.constrainRegClass(ResultReg, RC);
4250   if (Register::isVirtualRegister(SrcReg0))
4251     MRI.constrainRegClass(SrcReg0, RC);
4252   if (Register::isVirtualRegister(SrcReg1))
4253     MRI.constrainRegClass(SrcReg1, RC);
4254   if (Register::isVirtualRegister(SrcReg2))
4255     MRI.constrainRegClass(SrcReg2, RC);
4256 
4257   MachineInstrBuilder MIB;
4258   if (kind == FMAInstKind::Default)
4259     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4260               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4261               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4262               .addReg(SrcReg2, getKillRegState(Src2IsKill));
4263   else if (kind == FMAInstKind::Indexed)
4264     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4265               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4266               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4267               .addReg(SrcReg1, getKillRegState(Src1IsKill))
4268               .addImm(MUL->getOperand(3).getImm());
4269   else if (kind == FMAInstKind::Accumulator)
4270     MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4271               .addReg(SrcReg2, getKillRegState(Src2IsKill))
4272               .addReg(SrcReg0, getKillRegState(Src0IsKill))
4273               .addReg(SrcReg1, getKillRegState(Src1IsKill));
4274   else
4275     assert(false && "Invalid FMA instruction kind \n");
4276   // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
4277   InsInstrs.push_back(MIB);
4278   return MUL;
4279 }
4280 
4281 /// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
4282 /// instructions.
4283 ///
4284 /// \see genFusedMultiply
4285 static MachineInstr *genFusedMultiplyAcc(
4286     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4287     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4288     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4289   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4290                           FMAInstKind::Accumulator);
4291 }
4292 
4293 /// genNeg - Helper to generate an intermediate negation of the second operand
4294 /// of Root
4295 static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
4296                        const TargetInstrInfo *TII, MachineInstr &Root,
4297                        SmallVectorImpl<MachineInstr *> &InsInstrs,
4298                        DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
4299                        unsigned MnegOpc, const TargetRegisterClass *RC) {
4300   Register NewVR = MRI.createVirtualRegister(RC);
4301   MachineInstrBuilder MIB =
4302       BuildMI(MF, Root.getDebugLoc(), TII->get(MnegOpc), NewVR)
4303           .add(Root.getOperand(2));
4304   InsInstrs.push_back(MIB);
4305 
4306   assert(InstrIdxForVirtReg.empty());
4307   InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4308 
4309   return NewVR;
4310 }
4311 
4312 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4313 /// instructions with an additional negation of the accumulator
4314 static MachineInstr *genFusedMultiplyAccNeg(
4315     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4316     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4317     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4318     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4319   assert(IdxMulOpd == 1);
4320 
4321   Register NewVR =
4322       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4323   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4324                           FMAInstKind::Accumulator, &NewVR);
4325 }
4326 
4327 /// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
4328 /// instructions.
4329 ///
4330 /// \see genFusedMultiply
4331 static MachineInstr *genFusedMultiplyIdx(
4332     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4333     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4334     unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
4335   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4336                           FMAInstKind::Indexed);
4337 }
4338 
4339 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
4340 /// instructions with an additional negation of the accumulator
4341 static MachineInstr *genFusedMultiplyIdxNeg(
4342     MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
4343     MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
4344     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
4345     unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
4346   assert(IdxMulOpd == 1);
4347 
4348   Register NewVR =
4349       genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
4350 
4351   return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
4352                           FMAInstKind::Indexed, &NewVR);
4353 }
4354 
4355 /// genMaddR - Generate madd instruction and combine mul and add using
4356 /// an extra virtual register
4357 /// Example - an ADD intermediate needs to be stored in a register:
4358 ///   MUL I=A,B,0
4359 ///   ADD R,I,Imm
4360 ///   ==> ORR  V, ZR, Imm
4361 ///   ==> MADD R,A,B,V
4362 /// \param MF Containing MachineFunction
4363 /// \param MRI Register information
4364 /// \param TII Target information
4365 /// \param Root is the ADD instruction
4366 /// \param [out] InsInstrs is a vector of machine instructions and will
4367 /// contain the generated madd instruction
4368 /// \param IdxMulOpd is index of operand in Root that is the result of
4369 /// the MUL. In the example above IdxMulOpd is 1.
4370 /// \param MaddOpc the opcode fo the madd instruction
4371 /// \param VR is a virtual register that holds the value of an ADD operand
4372 /// (V in the example above).
4373 /// \param RC Register class of operands
4374 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
4375                               const TargetInstrInfo *TII, MachineInstr &Root,
4376                               SmallVectorImpl<MachineInstr *> &InsInstrs,
4377                               unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
4378                               const TargetRegisterClass *RC) {
4379   assert(IdxMulOpd == 1 || IdxMulOpd == 2);
4380 
4381   MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
4382   Register ResultReg = Root.getOperand(0).getReg();
4383   Register SrcReg0 = MUL->getOperand(1).getReg();
4384   bool Src0IsKill = MUL->getOperand(1).isKill();
4385   Register SrcReg1 = MUL->getOperand(2).getReg();
4386   bool Src1IsKill = MUL->getOperand(2).isKill();
4387 
4388   if (Register::isVirtualRegister(ResultReg))
4389     MRI.constrainRegClass(ResultReg, RC);
4390   if (Register::isVirtualRegister(SrcReg0))
4391     MRI.constrainRegClass(SrcReg0, RC);
4392   if (Register::isVirtualRegister(SrcReg1))
4393     MRI.constrainRegClass(SrcReg1, RC);
4394   if (Register::isVirtualRegister(VR))
4395     MRI.constrainRegClass(VR, RC);
4396 
4397   MachineInstrBuilder MIB =
4398       BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
4399           .addReg(SrcReg0, getKillRegState(Src0IsKill))
4400           .addReg(SrcReg1, getKillRegState(Src1IsKill))
4401           .addReg(VR);
4402   // Insert the MADD
4403   InsInstrs.push_back(MIB);
4404   return MUL;
4405 }
4406 
4407 /// When getMachineCombinerPatterns() finds potential patterns,
4408 /// this function generates the instructions that could replace the
4409 /// original code sequence
4410 void AArch64InstrInfo::genAlternativeCodeSequence(
4411     MachineInstr &Root, MachineCombinerPattern Pattern,
4412     SmallVectorImpl<MachineInstr *> &InsInstrs,
4413     SmallVectorImpl<MachineInstr *> &DelInstrs,
4414     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
4415   MachineBasicBlock &MBB = *Root.getParent();
4416   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4417   MachineFunction &MF = *MBB.getParent();
4418   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
4419 
4420   MachineInstr *MUL;
4421   const TargetRegisterClass *RC;
4422   unsigned Opc;
4423   switch (Pattern) {
4424   default:
4425     // Reassociate instructions.
4426     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
4427                                                 DelInstrs, InstrIdxForVirtReg);
4428     return;
4429   case MachineCombinerPattern::MULADDW_OP1:
4430   case MachineCombinerPattern::MULADDX_OP1:
4431     // MUL I=A,B,0
4432     // ADD R,I,C
4433     // ==> MADD R,A,B,C
4434     // --- Create(MADD);
4435     if (Pattern == MachineCombinerPattern::MULADDW_OP1) {
4436       Opc = AArch64::MADDWrrr;
4437       RC = &AArch64::GPR32RegClass;
4438     } else {
4439       Opc = AArch64::MADDXrrr;
4440       RC = &AArch64::GPR64RegClass;
4441     }
4442     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4443     break;
4444   case MachineCombinerPattern::MULADDW_OP2:
4445   case MachineCombinerPattern::MULADDX_OP2:
4446     // MUL I=A,B,0
4447     // ADD R,C,I
4448     // ==> MADD R,A,B,C
4449     // --- Create(MADD);
4450     if (Pattern == MachineCombinerPattern::MULADDW_OP2) {
4451       Opc = AArch64::MADDWrrr;
4452       RC = &AArch64::GPR32RegClass;
4453     } else {
4454       Opc = AArch64::MADDXrrr;
4455       RC = &AArch64::GPR64RegClass;
4456     }
4457     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4458     break;
4459   case MachineCombinerPattern::MULADDWI_OP1:
4460   case MachineCombinerPattern::MULADDXI_OP1: {
4461     // MUL I=A,B,0
4462     // ADD R,I,Imm
4463     // ==> ORR  V, ZR, Imm
4464     // ==> MADD R,A,B,V
4465     // --- Create(MADD);
4466     const TargetRegisterClass *OrrRC;
4467     unsigned BitSize, OrrOpc, ZeroReg;
4468     if (Pattern == MachineCombinerPattern::MULADDWI_OP1) {
4469       OrrOpc = AArch64::ORRWri;
4470       OrrRC = &AArch64::GPR32spRegClass;
4471       BitSize = 32;
4472       ZeroReg = AArch64::WZR;
4473       Opc = AArch64::MADDWrrr;
4474       RC = &AArch64::GPR32RegClass;
4475     } else {
4476       OrrOpc = AArch64::ORRXri;
4477       OrrRC = &AArch64::GPR64spRegClass;
4478       BitSize = 64;
4479       ZeroReg = AArch64::XZR;
4480       Opc = AArch64::MADDXrrr;
4481       RC = &AArch64::GPR64RegClass;
4482     }
4483     Register NewVR = MRI.createVirtualRegister(OrrRC);
4484     uint64_t Imm = Root.getOperand(2).getImm();
4485 
4486     if (Root.getOperand(3).isImm()) {
4487       unsigned Val = Root.getOperand(3).getImm();
4488       Imm = Imm << Val;
4489     }
4490     uint64_t UImm = SignExtend64(Imm, BitSize);
4491     uint64_t Encoding;
4492     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4493       MachineInstrBuilder MIB1 =
4494           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4495               .addReg(ZeroReg)
4496               .addImm(Encoding);
4497       InsInstrs.push_back(MIB1);
4498       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4499       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4500     }
4501     break;
4502   }
4503   case MachineCombinerPattern::MULSUBW_OP1:
4504   case MachineCombinerPattern::MULSUBX_OP1: {
4505     // MUL I=A,B,0
4506     // SUB R,I, C
4507     // ==> SUB  V, 0, C
4508     // ==> MADD R,A,B,V // = -C + A*B
4509     // --- Create(MADD);
4510     const TargetRegisterClass *SubRC;
4511     unsigned SubOpc, ZeroReg;
4512     if (Pattern == MachineCombinerPattern::MULSUBW_OP1) {
4513       SubOpc = AArch64::SUBWrr;
4514       SubRC = &AArch64::GPR32spRegClass;
4515       ZeroReg = AArch64::WZR;
4516       Opc = AArch64::MADDWrrr;
4517       RC = &AArch64::GPR32RegClass;
4518     } else {
4519       SubOpc = AArch64::SUBXrr;
4520       SubRC = &AArch64::GPR64spRegClass;
4521       ZeroReg = AArch64::XZR;
4522       Opc = AArch64::MADDXrrr;
4523       RC = &AArch64::GPR64RegClass;
4524     }
4525     Register NewVR = MRI.createVirtualRegister(SubRC);
4526     // SUB NewVR, 0, C
4527     MachineInstrBuilder MIB1 =
4528         BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
4529             .addReg(ZeroReg)
4530             .add(Root.getOperand(2));
4531     InsInstrs.push_back(MIB1);
4532     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4533     MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4534     break;
4535   }
4536   case MachineCombinerPattern::MULSUBW_OP2:
4537   case MachineCombinerPattern::MULSUBX_OP2:
4538     // MUL I=A,B,0
4539     // SUB R,C,I
4540     // ==> MSUB R,A,B,C (computes C - A*B)
4541     // --- Create(MSUB);
4542     if (Pattern == MachineCombinerPattern::MULSUBW_OP2) {
4543       Opc = AArch64::MSUBWrrr;
4544       RC = &AArch64::GPR32RegClass;
4545     } else {
4546       Opc = AArch64::MSUBXrrr;
4547       RC = &AArch64::GPR64RegClass;
4548     }
4549     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4550     break;
4551   case MachineCombinerPattern::MULSUBWI_OP1:
4552   case MachineCombinerPattern::MULSUBXI_OP1: {
4553     // MUL I=A,B,0
4554     // SUB R,I, Imm
4555     // ==> ORR  V, ZR, -Imm
4556     // ==> MADD R,A,B,V // = -Imm + A*B
4557     // --- Create(MADD);
4558     const TargetRegisterClass *OrrRC;
4559     unsigned BitSize, OrrOpc, ZeroReg;
4560     if (Pattern == MachineCombinerPattern::MULSUBWI_OP1) {
4561       OrrOpc = AArch64::ORRWri;
4562       OrrRC = &AArch64::GPR32spRegClass;
4563       BitSize = 32;
4564       ZeroReg = AArch64::WZR;
4565       Opc = AArch64::MADDWrrr;
4566       RC = &AArch64::GPR32RegClass;
4567     } else {
4568       OrrOpc = AArch64::ORRXri;
4569       OrrRC = &AArch64::GPR64spRegClass;
4570       BitSize = 64;
4571       ZeroReg = AArch64::XZR;
4572       Opc = AArch64::MADDXrrr;
4573       RC = &AArch64::GPR64RegClass;
4574     }
4575     Register NewVR = MRI.createVirtualRegister(OrrRC);
4576     uint64_t Imm = Root.getOperand(2).getImm();
4577     if (Root.getOperand(3).isImm()) {
4578       unsigned Val = Root.getOperand(3).getImm();
4579       Imm = Imm << Val;
4580     }
4581     uint64_t UImm = SignExtend64(-Imm, BitSize);
4582     uint64_t Encoding;
4583     if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
4584       MachineInstrBuilder MIB1 =
4585           BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
4586               .addReg(ZeroReg)
4587               .addImm(Encoding);
4588       InsInstrs.push_back(MIB1);
4589       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
4590       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
4591     }
4592     break;
4593   }
4594 
4595   case MachineCombinerPattern::MULADDv8i8_OP1:
4596     Opc = AArch64::MLAv8i8;
4597     RC = &AArch64::FPR64RegClass;
4598     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4599     break;
4600   case MachineCombinerPattern::MULADDv8i8_OP2:
4601     Opc = AArch64::MLAv8i8;
4602     RC = &AArch64::FPR64RegClass;
4603     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4604     break;
4605   case MachineCombinerPattern::MULADDv16i8_OP1:
4606     Opc = AArch64::MLAv16i8;
4607     RC = &AArch64::FPR128RegClass;
4608     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4609     break;
4610   case MachineCombinerPattern::MULADDv16i8_OP2:
4611     Opc = AArch64::MLAv16i8;
4612     RC = &AArch64::FPR128RegClass;
4613     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4614     break;
4615   case MachineCombinerPattern::MULADDv4i16_OP1:
4616     Opc = AArch64::MLAv4i16;
4617     RC = &AArch64::FPR64RegClass;
4618     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4619     break;
4620   case MachineCombinerPattern::MULADDv4i16_OP2:
4621     Opc = AArch64::MLAv4i16;
4622     RC = &AArch64::FPR64RegClass;
4623     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4624     break;
4625   case MachineCombinerPattern::MULADDv8i16_OP1:
4626     Opc = AArch64::MLAv8i16;
4627     RC = &AArch64::FPR128RegClass;
4628     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4629     break;
4630   case MachineCombinerPattern::MULADDv8i16_OP2:
4631     Opc = AArch64::MLAv8i16;
4632     RC = &AArch64::FPR128RegClass;
4633     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4634     break;
4635   case MachineCombinerPattern::MULADDv2i32_OP1:
4636     Opc = AArch64::MLAv2i32;
4637     RC = &AArch64::FPR64RegClass;
4638     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4639     break;
4640   case MachineCombinerPattern::MULADDv2i32_OP2:
4641     Opc = AArch64::MLAv2i32;
4642     RC = &AArch64::FPR64RegClass;
4643     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4644     break;
4645   case MachineCombinerPattern::MULADDv4i32_OP1:
4646     Opc = AArch64::MLAv4i32;
4647     RC = &AArch64::FPR128RegClass;
4648     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4649     break;
4650   case MachineCombinerPattern::MULADDv4i32_OP2:
4651     Opc = AArch64::MLAv4i32;
4652     RC = &AArch64::FPR128RegClass;
4653     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4654     break;
4655 
4656   case MachineCombinerPattern::MULSUBv8i8_OP1:
4657     Opc = AArch64::MLAv8i8;
4658     RC = &AArch64::FPR64RegClass;
4659     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4660                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
4661                                  RC);
4662     break;
4663   case MachineCombinerPattern::MULSUBv8i8_OP2:
4664     Opc = AArch64::MLSv8i8;
4665     RC = &AArch64::FPR64RegClass;
4666     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4667     break;
4668   case MachineCombinerPattern::MULSUBv16i8_OP1:
4669     Opc = AArch64::MLAv16i8;
4670     RC = &AArch64::FPR128RegClass;
4671     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4672                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
4673                                  RC);
4674     break;
4675   case MachineCombinerPattern::MULSUBv16i8_OP2:
4676     Opc = AArch64::MLSv16i8;
4677     RC = &AArch64::FPR128RegClass;
4678     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4679     break;
4680   case MachineCombinerPattern::MULSUBv4i16_OP1:
4681     Opc = AArch64::MLAv4i16;
4682     RC = &AArch64::FPR64RegClass;
4683     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4684                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4685                                  RC);
4686     break;
4687   case MachineCombinerPattern::MULSUBv4i16_OP2:
4688     Opc = AArch64::MLSv4i16;
4689     RC = &AArch64::FPR64RegClass;
4690     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4691     break;
4692   case MachineCombinerPattern::MULSUBv8i16_OP1:
4693     Opc = AArch64::MLAv8i16;
4694     RC = &AArch64::FPR128RegClass;
4695     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4696                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4697                                  RC);
4698     break;
4699   case MachineCombinerPattern::MULSUBv8i16_OP2:
4700     Opc = AArch64::MLSv8i16;
4701     RC = &AArch64::FPR128RegClass;
4702     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4703     break;
4704   case MachineCombinerPattern::MULSUBv2i32_OP1:
4705     Opc = AArch64::MLAv2i32;
4706     RC = &AArch64::FPR64RegClass;
4707     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4708                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4709                                  RC);
4710     break;
4711   case MachineCombinerPattern::MULSUBv2i32_OP2:
4712     Opc = AArch64::MLSv2i32;
4713     RC = &AArch64::FPR64RegClass;
4714     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4715     break;
4716   case MachineCombinerPattern::MULSUBv4i32_OP1:
4717     Opc = AArch64::MLAv4i32;
4718     RC = &AArch64::FPR128RegClass;
4719     MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
4720                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
4721                                  RC);
4722     break;
4723   case MachineCombinerPattern::MULSUBv4i32_OP2:
4724     Opc = AArch64::MLSv4i32;
4725     RC = &AArch64::FPR128RegClass;
4726     MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4727     break;
4728 
4729   case MachineCombinerPattern::MULADDv4i16_indexed_OP1:
4730     Opc = AArch64::MLAv4i16_indexed;
4731     RC = &AArch64::FPR64RegClass;
4732     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4733     break;
4734   case MachineCombinerPattern::MULADDv4i16_indexed_OP2:
4735     Opc = AArch64::MLAv4i16_indexed;
4736     RC = &AArch64::FPR64RegClass;
4737     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4738     break;
4739   case MachineCombinerPattern::MULADDv8i16_indexed_OP1:
4740     Opc = AArch64::MLAv8i16_indexed;
4741     RC = &AArch64::FPR128RegClass;
4742     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4743     break;
4744   case MachineCombinerPattern::MULADDv8i16_indexed_OP2:
4745     Opc = AArch64::MLAv8i16_indexed;
4746     RC = &AArch64::FPR128RegClass;
4747     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4748     break;
4749   case MachineCombinerPattern::MULADDv2i32_indexed_OP1:
4750     Opc = AArch64::MLAv2i32_indexed;
4751     RC = &AArch64::FPR64RegClass;
4752     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4753     break;
4754   case MachineCombinerPattern::MULADDv2i32_indexed_OP2:
4755     Opc = AArch64::MLAv2i32_indexed;
4756     RC = &AArch64::FPR64RegClass;
4757     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4758     break;
4759   case MachineCombinerPattern::MULADDv4i32_indexed_OP1:
4760     Opc = AArch64::MLAv4i32_indexed;
4761     RC = &AArch64::FPR128RegClass;
4762     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4763     break;
4764   case MachineCombinerPattern::MULADDv4i32_indexed_OP2:
4765     Opc = AArch64::MLAv4i32_indexed;
4766     RC = &AArch64::FPR128RegClass;
4767     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4768     break;
4769 
4770   case MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
4771     Opc = AArch64::MLAv4i16_indexed;
4772     RC = &AArch64::FPR64RegClass;
4773     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4774                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
4775                                  RC);
4776     break;
4777   case MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
4778     Opc = AArch64::MLSv4i16_indexed;
4779     RC = &AArch64::FPR64RegClass;
4780     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4781     break;
4782   case MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
4783     Opc = AArch64::MLAv8i16_indexed;
4784     RC = &AArch64::FPR128RegClass;
4785     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4786                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
4787                                  RC);
4788     break;
4789   case MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
4790     Opc = AArch64::MLSv8i16_indexed;
4791     RC = &AArch64::FPR128RegClass;
4792     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4793     break;
4794   case MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
4795     Opc = AArch64::MLAv2i32_indexed;
4796     RC = &AArch64::FPR64RegClass;
4797     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4798                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
4799                                  RC);
4800     break;
4801   case MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
4802     Opc = AArch64::MLSv2i32_indexed;
4803     RC = &AArch64::FPR64RegClass;
4804     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4805     break;
4806   case MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
4807     Opc = AArch64::MLAv4i32_indexed;
4808     RC = &AArch64::FPR128RegClass;
4809     MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
4810                                  InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
4811                                  RC);
4812     break;
4813   case MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
4814     Opc = AArch64::MLSv4i32_indexed;
4815     RC = &AArch64::FPR128RegClass;
4816     MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4817     break;
4818 
4819   // Floating Point Support
4820   case MachineCombinerPattern::FMULADDH_OP1:
4821     Opc = AArch64::FMADDHrrr;
4822     RC = &AArch64::FPR16RegClass;
4823     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4824     break;
4825   case MachineCombinerPattern::FMULADDS_OP1:
4826     Opc = AArch64::FMADDSrrr;
4827     RC = &AArch64::FPR32RegClass;
4828     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4829     break;
4830   case MachineCombinerPattern::FMULADDD_OP1:
4831     Opc = AArch64::FMADDDrrr;
4832     RC = &AArch64::FPR64RegClass;
4833     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
4834     break;
4835 
4836   case MachineCombinerPattern::FMULADDH_OP2:
4837     Opc = AArch64::FMADDHrrr;
4838     RC = &AArch64::FPR16RegClass;
4839     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4840     break;
4841   case MachineCombinerPattern::FMULADDS_OP2:
4842     Opc = AArch64::FMADDSrrr;
4843     RC = &AArch64::FPR32RegClass;
4844     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4845     break;
4846   case MachineCombinerPattern::FMULADDD_OP2:
4847     Opc = AArch64::FMADDDrrr;
4848     RC = &AArch64::FPR64RegClass;
4849     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
4850     break;
4851 
4852   case MachineCombinerPattern::FMLAv1i32_indexed_OP1:
4853     Opc = AArch64::FMLAv1i32_indexed;
4854     RC = &AArch64::FPR32RegClass;
4855     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4856                            FMAInstKind::Indexed);
4857     break;
4858   case MachineCombinerPattern::FMLAv1i32_indexed_OP2:
4859     Opc = AArch64::FMLAv1i32_indexed;
4860     RC = &AArch64::FPR32RegClass;
4861     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4862                            FMAInstKind::Indexed);
4863     break;
4864 
4865   case MachineCombinerPattern::FMLAv1i64_indexed_OP1:
4866     Opc = AArch64::FMLAv1i64_indexed;
4867     RC = &AArch64::FPR64RegClass;
4868     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4869                            FMAInstKind::Indexed);
4870     break;
4871   case MachineCombinerPattern::FMLAv1i64_indexed_OP2:
4872     Opc = AArch64::FMLAv1i64_indexed;
4873     RC = &AArch64::FPR64RegClass;
4874     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4875                            FMAInstKind::Indexed);
4876     break;
4877 
4878   case MachineCombinerPattern::FMLAv4i16_indexed_OP1:
4879     RC = &AArch64::FPR64RegClass;
4880     Opc = AArch64::FMLAv4i16_indexed;
4881     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4882                            FMAInstKind::Indexed);
4883     break;
4884   case MachineCombinerPattern::FMLAv4f16_OP1:
4885     RC = &AArch64::FPR64RegClass;
4886     Opc = AArch64::FMLAv4f16;
4887     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4888                            FMAInstKind::Accumulator);
4889     break;
4890   case MachineCombinerPattern::FMLAv4i16_indexed_OP2:
4891     RC = &AArch64::FPR64RegClass;
4892     Opc = AArch64::FMLAv4i16_indexed;
4893     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4894                            FMAInstKind::Indexed);
4895     break;
4896   case MachineCombinerPattern::FMLAv4f16_OP2:
4897     RC = &AArch64::FPR64RegClass;
4898     Opc = AArch64::FMLAv4f16;
4899     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4900                            FMAInstKind::Accumulator);
4901     break;
4902 
4903   case MachineCombinerPattern::FMLAv2i32_indexed_OP1:
4904   case MachineCombinerPattern::FMLAv2f32_OP1:
4905     RC = &AArch64::FPR64RegClass;
4906     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
4907       Opc = AArch64::FMLAv2i32_indexed;
4908       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4909                              FMAInstKind::Indexed);
4910     } else {
4911       Opc = AArch64::FMLAv2f32;
4912       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4913                              FMAInstKind::Accumulator);
4914     }
4915     break;
4916   case MachineCombinerPattern::FMLAv2i32_indexed_OP2:
4917   case MachineCombinerPattern::FMLAv2f32_OP2:
4918     RC = &AArch64::FPR64RegClass;
4919     if (Pattern == MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
4920       Opc = AArch64::FMLAv2i32_indexed;
4921       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4922                              FMAInstKind::Indexed);
4923     } else {
4924       Opc = AArch64::FMLAv2f32;
4925       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4926                              FMAInstKind::Accumulator);
4927     }
4928     break;
4929 
4930   case MachineCombinerPattern::FMLAv8i16_indexed_OP1:
4931     RC = &AArch64::FPR128RegClass;
4932     Opc = AArch64::FMLAv8i16_indexed;
4933     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4934                            FMAInstKind::Indexed);
4935     break;
4936   case MachineCombinerPattern::FMLAv8f16_OP1:
4937     RC = &AArch64::FPR128RegClass;
4938     Opc = AArch64::FMLAv8f16;
4939     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4940                            FMAInstKind::Accumulator);
4941     break;
4942   case MachineCombinerPattern::FMLAv8i16_indexed_OP2:
4943     RC = &AArch64::FPR128RegClass;
4944     Opc = AArch64::FMLAv8i16_indexed;
4945     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4946                            FMAInstKind::Indexed);
4947     break;
4948   case MachineCombinerPattern::FMLAv8f16_OP2:
4949     RC = &AArch64::FPR128RegClass;
4950     Opc = AArch64::FMLAv8f16;
4951     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4952                            FMAInstKind::Accumulator);
4953     break;
4954 
4955   case MachineCombinerPattern::FMLAv2i64_indexed_OP1:
4956   case MachineCombinerPattern::FMLAv2f64_OP1:
4957     RC = &AArch64::FPR128RegClass;
4958     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
4959       Opc = AArch64::FMLAv2i64_indexed;
4960       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4961                              FMAInstKind::Indexed);
4962     } else {
4963       Opc = AArch64::FMLAv2f64;
4964       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4965                              FMAInstKind::Accumulator);
4966     }
4967     break;
4968   case MachineCombinerPattern::FMLAv2i64_indexed_OP2:
4969   case MachineCombinerPattern::FMLAv2f64_OP2:
4970     RC = &AArch64::FPR128RegClass;
4971     if (Pattern == MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
4972       Opc = AArch64::FMLAv2i64_indexed;
4973       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4974                              FMAInstKind::Indexed);
4975     } else {
4976       Opc = AArch64::FMLAv2f64;
4977       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
4978                              FMAInstKind::Accumulator);
4979     }
4980     break;
4981 
4982   case MachineCombinerPattern::FMLAv4i32_indexed_OP1:
4983   case MachineCombinerPattern::FMLAv4f32_OP1:
4984     RC = &AArch64::FPR128RegClass;
4985     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
4986       Opc = AArch64::FMLAv4i32_indexed;
4987       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4988                              FMAInstKind::Indexed);
4989     } else {
4990       Opc = AArch64::FMLAv4f32;
4991       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
4992                              FMAInstKind::Accumulator);
4993     }
4994     break;
4995 
4996   case MachineCombinerPattern::FMLAv4i32_indexed_OP2:
4997   case MachineCombinerPattern::FMLAv4f32_OP2:
4998     RC = &AArch64::FPR128RegClass;
4999     if (Pattern == MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
5000       Opc = AArch64::FMLAv4i32_indexed;
5001       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5002                              FMAInstKind::Indexed);
5003     } else {
5004       Opc = AArch64::FMLAv4f32;
5005       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5006                              FMAInstKind::Accumulator);
5007     }
5008     break;
5009 
5010   case MachineCombinerPattern::FMULSUBH_OP1:
5011     Opc = AArch64::FNMSUBHrrr;
5012     RC = &AArch64::FPR16RegClass;
5013     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5014     break;
5015   case MachineCombinerPattern::FMULSUBS_OP1:
5016     Opc = AArch64::FNMSUBSrrr;
5017     RC = &AArch64::FPR32RegClass;
5018     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5019     break;
5020   case MachineCombinerPattern::FMULSUBD_OP1:
5021     Opc = AArch64::FNMSUBDrrr;
5022     RC = &AArch64::FPR64RegClass;
5023     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5024     break;
5025 
5026   case MachineCombinerPattern::FNMULSUBH_OP1:
5027     Opc = AArch64::FNMADDHrrr;
5028     RC = &AArch64::FPR16RegClass;
5029     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5030     break;
5031   case MachineCombinerPattern::FNMULSUBS_OP1:
5032     Opc = AArch64::FNMADDSrrr;
5033     RC = &AArch64::FPR32RegClass;
5034     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5035     break;
5036   case MachineCombinerPattern::FNMULSUBD_OP1:
5037     Opc = AArch64::FNMADDDrrr;
5038     RC = &AArch64::FPR64RegClass;
5039     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
5040     break;
5041 
5042   case MachineCombinerPattern::FMULSUBH_OP2:
5043     Opc = AArch64::FMSUBHrrr;
5044     RC = &AArch64::FPR16RegClass;
5045     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5046     break;
5047   case MachineCombinerPattern::FMULSUBS_OP2:
5048     Opc = AArch64::FMSUBSrrr;
5049     RC = &AArch64::FPR32RegClass;
5050     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5051     break;
5052   case MachineCombinerPattern::FMULSUBD_OP2:
5053     Opc = AArch64::FMSUBDrrr;
5054     RC = &AArch64::FPR64RegClass;
5055     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
5056     break;
5057 
5058   case MachineCombinerPattern::FMLSv1i32_indexed_OP2:
5059     Opc = AArch64::FMLSv1i32_indexed;
5060     RC = &AArch64::FPR32RegClass;
5061     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5062                            FMAInstKind::Indexed);
5063     break;
5064 
5065   case MachineCombinerPattern::FMLSv1i64_indexed_OP2:
5066     Opc = AArch64::FMLSv1i64_indexed;
5067     RC = &AArch64::FPR64RegClass;
5068     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5069                            FMAInstKind::Indexed);
5070     break;
5071 
5072   case MachineCombinerPattern::FMLSv4f16_OP1:
5073   case MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
5074     RC = &AArch64::FPR64RegClass;
5075     Register NewVR = MRI.createVirtualRegister(RC);
5076     MachineInstrBuilder MIB1 =
5077         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f16), NewVR)
5078             .add(Root.getOperand(2));
5079     InsInstrs.push_back(MIB1);
5080     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5081     if (Pattern == MachineCombinerPattern::FMLSv4f16_OP1) {
5082       Opc = AArch64::FMLAv4f16;
5083       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5084                              FMAInstKind::Accumulator, &NewVR);
5085     } else {
5086       Opc = AArch64::FMLAv4i16_indexed;
5087       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5088                              FMAInstKind::Indexed, &NewVR);
5089     }
5090     break;
5091   }
5092   case MachineCombinerPattern::FMLSv4f16_OP2:
5093     RC = &AArch64::FPR64RegClass;
5094     Opc = AArch64::FMLSv4f16;
5095     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5096                            FMAInstKind::Accumulator);
5097     break;
5098   case MachineCombinerPattern::FMLSv4i16_indexed_OP2:
5099     RC = &AArch64::FPR64RegClass;
5100     Opc = AArch64::FMLSv4i16_indexed;
5101     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5102                            FMAInstKind::Indexed);
5103     break;
5104 
5105   case MachineCombinerPattern::FMLSv2f32_OP2:
5106   case MachineCombinerPattern::FMLSv2i32_indexed_OP2:
5107     RC = &AArch64::FPR64RegClass;
5108     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
5109       Opc = AArch64::FMLSv2i32_indexed;
5110       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5111                              FMAInstKind::Indexed);
5112     } else {
5113       Opc = AArch64::FMLSv2f32;
5114       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5115                              FMAInstKind::Accumulator);
5116     }
5117     break;
5118 
5119   case MachineCombinerPattern::FMLSv8f16_OP1:
5120   case MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
5121     RC = &AArch64::FPR128RegClass;
5122     Register NewVR = MRI.createVirtualRegister(RC);
5123     MachineInstrBuilder MIB1 =
5124         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv8f16), NewVR)
5125             .add(Root.getOperand(2));
5126     InsInstrs.push_back(MIB1);
5127     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5128     if (Pattern == MachineCombinerPattern::FMLSv8f16_OP1) {
5129       Opc = AArch64::FMLAv8f16;
5130       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5131                              FMAInstKind::Accumulator, &NewVR);
5132     } else {
5133       Opc = AArch64::FMLAv8i16_indexed;
5134       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5135                              FMAInstKind::Indexed, &NewVR);
5136     }
5137     break;
5138   }
5139   case MachineCombinerPattern::FMLSv8f16_OP2:
5140     RC = &AArch64::FPR128RegClass;
5141     Opc = AArch64::FMLSv8f16;
5142     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5143                            FMAInstKind::Accumulator);
5144     break;
5145   case MachineCombinerPattern::FMLSv8i16_indexed_OP2:
5146     RC = &AArch64::FPR128RegClass;
5147     Opc = AArch64::FMLSv8i16_indexed;
5148     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5149                            FMAInstKind::Indexed);
5150     break;
5151 
5152   case MachineCombinerPattern::FMLSv2f64_OP2:
5153   case MachineCombinerPattern::FMLSv2i64_indexed_OP2:
5154     RC = &AArch64::FPR128RegClass;
5155     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
5156       Opc = AArch64::FMLSv2i64_indexed;
5157       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5158                              FMAInstKind::Indexed);
5159     } else {
5160       Opc = AArch64::FMLSv2f64;
5161       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5162                              FMAInstKind::Accumulator);
5163     }
5164     break;
5165 
5166   case MachineCombinerPattern::FMLSv4f32_OP2:
5167   case MachineCombinerPattern::FMLSv4i32_indexed_OP2:
5168     RC = &AArch64::FPR128RegClass;
5169     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
5170       Opc = AArch64::FMLSv4i32_indexed;
5171       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5172                              FMAInstKind::Indexed);
5173     } else {
5174       Opc = AArch64::FMLSv4f32;
5175       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
5176                              FMAInstKind::Accumulator);
5177     }
5178     break;
5179   case MachineCombinerPattern::FMLSv2f32_OP1:
5180   case MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
5181     RC = &AArch64::FPR64RegClass;
5182     Register NewVR = MRI.createVirtualRegister(RC);
5183     MachineInstrBuilder MIB1 =
5184         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f32), NewVR)
5185             .add(Root.getOperand(2));
5186     InsInstrs.push_back(MIB1);
5187     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5188     if (Pattern == MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
5189       Opc = AArch64::FMLAv2i32_indexed;
5190       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5191                              FMAInstKind::Indexed, &NewVR);
5192     } else {
5193       Opc = AArch64::FMLAv2f32;
5194       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5195                              FMAInstKind::Accumulator, &NewVR);
5196     }
5197     break;
5198   }
5199   case MachineCombinerPattern::FMLSv4f32_OP1:
5200   case MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
5201     RC = &AArch64::FPR128RegClass;
5202     Register NewVR = MRI.createVirtualRegister(RC);
5203     MachineInstrBuilder MIB1 =
5204         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv4f32), NewVR)
5205             .add(Root.getOperand(2));
5206     InsInstrs.push_back(MIB1);
5207     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5208     if (Pattern == MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
5209       Opc = AArch64::FMLAv4i32_indexed;
5210       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5211                              FMAInstKind::Indexed, &NewVR);
5212     } else {
5213       Opc = AArch64::FMLAv4f32;
5214       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5215                              FMAInstKind::Accumulator, &NewVR);
5216     }
5217     break;
5218   }
5219   case MachineCombinerPattern::FMLSv2f64_OP1:
5220   case MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
5221     RC = &AArch64::FPR128RegClass;
5222     Register NewVR = MRI.createVirtualRegister(RC);
5223     MachineInstrBuilder MIB1 =
5224         BuildMI(MF, Root.getDebugLoc(), TII->get(AArch64::FNEGv2f64), NewVR)
5225             .add(Root.getOperand(2));
5226     InsInstrs.push_back(MIB1);
5227     InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
5228     if (Pattern == MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
5229       Opc = AArch64::FMLAv2i64_indexed;
5230       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5231                              FMAInstKind::Indexed, &NewVR);
5232     } else {
5233       Opc = AArch64::FMLAv2f64;
5234       MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
5235                              FMAInstKind::Accumulator, &NewVR);
5236     }
5237     break;
5238   }
5239   } // end switch (Pattern)
5240   // Record MUL and ADD/SUB for deletion
5241   DelInstrs.push_back(MUL);
5242   DelInstrs.push_back(&Root);
5243 }
5244 
5245 /// Replace csincr-branch sequence by simple conditional branch
5246 ///
5247 /// Examples:
5248 /// 1. \code
5249 ///   csinc  w9, wzr, wzr, <condition code>
5250 ///   tbnz   w9, #0, 0x44
5251 ///    \endcode
5252 /// to
5253 ///    \code
5254 ///   b.<inverted condition code>
5255 ///    \endcode
5256 ///
5257 /// 2. \code
5258 ///   csinc w9, wzr, wzr, <condition code>
5259 ///   tbz   w9, #0, 0x44
5260 ///    \endcode
5261 /// to
5262 ///    \code
5263 ///   b.<condition code>
5264 ///    \endcode
5265 ///
5266 /// Replace compare and branch sequence by TBZ/TBNZ instruction when the
5267 /// compare's constant operand is power of 2.
5268 ///
5269 /// Examples:
5270 ///    \code
5271 ///   and  w8, w8, #0x400
5272 ///   cbnz w8, L1
5273 ///    \endcode
5274 /// to
5275 ///    \code
5276 ///   tbnz w8, #10, L1
5277 ///    \endcode
5278 ///
5279 /// \param  MI Conditional Branch
5280 /// \return True when the simple conditional branch is generated
5281 ///
5282 bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
5283   bool IsNegativeBranch = false;
5284   bool IsTestAndBranch = false;
5285   unsigned TargetBBInMI = 0;
5286   switch (MI.getOpcode()) {
5287   default:
5288     llvm_unreachable("Unknown branch instruction?");
5289   case AArch64::Bcc:
5290     return false;
5291   case AArch64::CBZW:
5292   case AArch64::CBZX:
5293     TargetBBInMI = 1;
5294     break;
5295   case AArch64::CBNZW:
5296   case AArch64::CBNZX:
5297     TargetBBInMI = 1;
5298     IsNegativeBranch = true;
5299     break;
5300   case AArch64::TBZW:
5301   case AArch64::TBZX:
5302     TargetBBInMI = 2;
5303     IsTestAndBranch = true;
5304     break;
5305   case AArch64::TBNZW:
5306   case AArch64::TBNZX:
5307     TargetBBInMI = 2;
5308     IsNegativeBranch = true;
5309     IsTestAndBranch = true;
5310     break;
5311   }
5312   // So we increment a zero register and test for bits other
5313   // than bit 0? Conservatively bail out in case the verifier
5314   // missed this case.
5315   if (IsTestAndBranch && MI.getOperand(1).getImm())
5316     return false;
5317 
5318   // Find Definition.
5319   assert(MI.getParent() && "Incomplete machine instruciton\n");
5320   MachineBasicBlock *MBB = MI.getParent();
5321   MachineFunction *MF = MBB->getParent();
5322   MachineRegisterInfo *MRI = &MF->getRegInfo();
5323   Register VReg = MI.getOperand(0).getReg();
5324   if (!Register::isVirtualRegister(VReg))
5325     return false;
5326 
5327   MachineInstr *DefMI = MRI->getVRegDef(VReg);
5328 
5329   // Look through COPY instructions to find definition.
5330   while (DefMI->isCopy()) {
5331     Register CopyVReg = DefMI->getOperand(1).getReg();
5332     if (!MRI->hasOneNonDBGUse(CopyVReg))
5333       return false;
5334     if (!MRI->hasOneDef(CopyVReg))
5335       return false;
5336     DefMI = MRI->getVRegDef(CopyVReg);
5337   }
5338 
5339   switch (DefMI->getOpcode()) {
5340   default:
5341     return false;
5342   // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
5343   case AArch64::ANDWri:
5344   case AArch64::ANDXri: {
5345     if (IsTestAndBranch)
5346       return false;
5347     if (DefMI->getParent() != MBB)
5348       return false;
5349     if (!MRI->hasOneNonDBGUse(VReg))
5350       return false;
5351 
5352     bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
5353     uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
5354         DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
5355     if (!isPowerOf2_64(Mask))
5356       return false;
5357 
5358     MachineOperand &MO = DefMI->getOperand(1);
5359     Register NewReg = MO.getReg();
5360     if (!Register::isVirtualRegister(NewReg))
5361       return false;
5362 
5363     assert(!MRI->def_empty(NewReg) && "Register must be defined.");
5364 
5365     MachineBasicBlock &RefToMBB = *MBB;
5366     MachineBasicBlock *TBB = MI.getOperand(1).getMBB();
5367     DebugLoc DL = MI.getDebugLoc();
5368     unsigned Imm = Log2_64(Mask);
5369     unsigned Opc = (Imm < 32)
5370                        ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
5371                        : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
5372     MachineInstr *NewMI = BuildMI(RefToMBB, MI, DL, get(Opc))
5373                               .addReg(NewReg)
5374                               .addImm(Imm)
5375                               .addMBB(TBB);
5376     // Register lives on to the CBZ now.
5377     MO.setIsKill(false);
5378 
5379     // For immediate smaller than 32, we need to use the 32-bit
5380     // variant (W) in all cases. Indeed the 64-bit variant does not
5381     // allow to encode them.
5382     // Therefore, if the input register is 64-bit, we need to take the
5383     // 32-bit sub-part.
5384     if (!Is32Bit && Imm < 32)
5385       NewMI->getOperand(0).setSubReg(AArch64::sub_32);
5386     MI.eraseFromParent();
5387     return true;
5388   }
5389   // Look for CSINC
5390   case AArch64::CSINCWr:
5391   case AArch64::CSINCXr: {
5392     if (!(DefMI->getOperand(1).getReg() == AArch64::WZR &&
5393           DefMI->getOperand(2).getReg() == AArch64::WZR) &&
5394         !(DefMI->getOperand(1).getReg() == AArch64::XZR &&
5395           DefMI->getOperand(2).getReg() == AArch64::XZR))
5396       return false;
5397 
5398     if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
5399       return false;
5400 
5401     AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(3).getImm();
5402     // Convert only when the condition code is not modified between
5403     // the CSINC and the branch. The CC may be used by other
5404     // instructions in between.
5405     if (areCFlagsAccessedBetweenInstrs(DefMI, MI, &getRegisterInfo(), AK_Write))
5406       return false;
5407     MachineBasicBlock &RefToMBB = *MBB;
5408     MachineBasicBlock *TBB = MI.getOperand(TargetBBInMI).getMBB();
5409     DebugLoc DL = MI.getDebugLoc();
5410     if (IsNegativeBranch)
5411       CC = AArch64CC::getInvertedCondCode(CC);
5412     BuildMI(RefToMBB, MI, DL, get(AArch64::Bcc)).addImm(CC).addMBB(TBB);
5413     MI.eraseFromParent();
5414     return true;
5415   }
5416   }
5417 }
5418 
5419 std::pair<unsigned, unsigned>
5420 AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5421   const unsigned Mask = AArch64II::MO_FRAGMENT;
5422   return std::make_pair(TF & Mask, TF & ~Mask);
5423 }
5424 
5425 ArrayRef<std::pair<unsigned, const char *>>
5426 AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5427   using namespace AArch64II;
5428 
5429   static const std::pair<unsigned, const char *> TargetFlags[] = {
5430       {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
5431       {MO_G3, "aarch64-g3"},     {MO_G2, "aarch64-g2"},
5432       {MO_G1, "aarch64-g1"},     {MO_G0, "aarch64-g0"},
5433       {MO_HI12, "aarch64-hi12"}};
5434   return makeArrayRef(TargetFlags);
5435 }
5436 
5437 ArrayRef<std::pair<unsigned, const char *>>
5438 AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5439   using namespace AArch64II;
5440 
5441   static const std::pair<unsigned, const char *> TargetFlags[] = {
5442       {MO_COFFSTUB, "aarch64-coffstub"},
5443       {MO_GOT, "aarch64-got"},
5444       {MO_NC, "aarch64-nc"},
5445       {MO_S, "aarch64-s"},
5446       {MO_TLS, "aarch64-tls"},
5447       {MO_DLLIMPORT, "aarch64-dllimport"},
5448       {MO_PREL, "aarch64-prel"},
5449       {MO_TAGGED, "aarch64-tagged"}};
5450   return makeArrayRef(TargetFlags);
5451 }
5452 
5453 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5454 AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5455   static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5456       {{MOSuppressPair, "aarch64-suppress-pair"},
5457        {MOStridedAccess, "aarch64-strided-access"}};
5458   return makeArrayRef(TargetFlags);
5459 }
5460 
5461 /// Constants defining how certain sequences should be outlined.
5462 /// This encompasses how an outlined function should be called, and what kind of
5463 /// frame should be emitted for that outlined function.
5464 ///
5465 /// \p MachineOutlinerDefault implies that the function should be called with
5466 /// a save and restore of LR to the stack.
5467 ///
5468 /// That is,
5469 ///
5470 /// I1     Save LR                    OUTLINED_FUNCTION:
5471 /// I2 --> BL OUTLINED_FUNCTION       I1
5472 /// I3     Restore LR                 I2
5473 ///                                   I3
5474 ///                                   RET
5475 ///
5476 /// * Call construction overhead: 3 (save + BL + restore)
5477 /// * Frame construction overhead: 1 (ret)
5478 /// * Requires stack fixups? Yes
5479 ///
5480 /// \p MachineOutlinerTailCall implies that the function is being created from
5481 /// a sequence of instructions ending in a return.
5482 ///
5483 /// That is,
5484 ///
5485 /// I1                             OUTLINED_FUNCTION:
5486 /// I2 --> B OUTLINED_FUNCTION     I1
5487 /// RET                            I2
5488 ///                                RET
5489 ///
5490 /// * Call construction overhead: 1 (B)
5491 /// * Frame construction overhead: 0 (Return included in sequence)
5492 /// * Requires stack fixups? No
5493 ///
5494 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5495 /// a BL instruction, but doesn't require LR to be saved and restored. This
5496 /// happens when LR is known to be dead.
5497 ///
5498 /// That is,
5499 ///
5500 /// I1                                OUTLINED_FUNCTION:
5501 /// I2 --> BL OUTLINED_FUNCTION       I1
5502 /// I3                                I2
5503 ///                                   I3
5504 ///                                   RET
5505 ///
5506 /// * Call construction overhead: 1 (BL)
5507 /// * Frame construction overhead: 1 (RET)
5508 /// * Requires stack fixups? No
5509 ///
5510 /// \p MachineOutlinerThunk implies that the function is being created from
5511 /// a sequence of instructions ending in a call. The outlined function is
5512 /// called with a BL instruction, and the outlined function tail-calls the
5513 /// original call destination.
5514 ///
5515 /// That is,
5516 ///
5517 /// I1                                OUTLINED_FUNCTION:
5518 /// I2 --> BL OUTLINED_FUNCTION       I1
5519 /// BL f                              I2
5520 ///                                   B f
5521 /// * Call construction overhead: 1 (BL)
5522 /// * Frame construction overhead: 0
5523 /// * Requires stack fixups? No
5524 ///
5525 /// \p MachineOutlinerRegSave implies that the function should be called with a
5526 /// save and restore of LR to an available register. This allows us to avoid
5527 /// stack fixups. Note that this outlining variant is compatible with the
5528 /// NoLRSave case.
5529 ///
5530 /// That is,
5531 ///
5532 /// I1     Save LR                    OUTLINED_FUNCTION:
5533 /// I2 --> BL OUTLINED_FUNCTION       I1
5534 /// I3     Restore LR                 I2
5535 ///                                   I3
5536 ///                                   RET
5537 ///
5538 /// * Call construction overhead: 3 (save + BL + restore)
5539 /// * Frame construction overhead: 1 (ret)
5540 /// * Requires stack fixups? No
5541 enum MachineOutlinerClass {
5542   MachineOutlinerDefault,  /// Emit a save, restore, call, and return.
5543   MachineOutlinerTailCall, /// Only emit a branch.
5544   MachineOutlinerNoLRSave, /// Emit a call and return.
5545   MachineOutlinerThunk,    /// Emit a call and tail-call.
5546   MachineOutlinerRegSave   /// Same as default, but save to a register.
5547 };
5548 
5549 enum MachineOutlinerMBBFlags {
5550   LRUnavailableSomewhere = 0x2,
5551   HasCalls = 0x4,
5552   UnsafeRegsDead = 0x8
5553 };
5554 
5555 unsigned
5556 AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5557   assert(C.LRUWasSet && "LRU wasn't set?");
5558   MachineFunction *MF = C.getMF();
5559   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
5560       MF->getSubtarget().getRegisterInfo());
5561 
5562   // Check if there is an available register across the sequence that we can
5563   // use.
5564   for (unsigned Reg : AArch64::GPR64RegClass) {
5565     if (!ARI->isReservedReg(*MF, Reg) &&
5566         Reg != AArch64::LR &&  // LR is not reserved, but don't use it.
5567         Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
5568         Reg != AArch64::X17 && // Ditto for X17.
5569         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5570       return Reg;
5571   }
5572 
5573   // No suitable register. Return 0.
5574   return 0u;
5575 }
5576 
5577 static bool
5578 outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
5579                                          const outliner::Candidate &b) {
5580   const Function &Fa = a.getMF()->getFunction();
5581   const Function &Fb = b.getMF()->getFunction();
5582 
5583   // If none of the functions have the "sign-return-address" attribute their
5584   // signing behaviour is equal
5585   if (!Fa.hasFnAttribute("sign-return-address") &&
5586       !Fb.hasFnAttribute("sign-return-address")) {
5587     return true;
5588   }
5589 
5590   // If both functions have the "sign-return-address" attribute their signing
5591   // behaviour is equal, if the values of the attributes are equal
5592   if (Fa.hasFnAttribute("sign-return-address") &&
5593       Fb.hasFnAttribute("sign-return-address")) {
5594     StringRef ScopeA =
5595         Fa.getFnAttribute("sign-return-address").getValueAsString();
5596     StringRef ScopeB =
5597         Fb.getFnAttribute("sign-return-address").getValueAsString();
5598     return ScopeA.equals(ScopeB);
5599   }
5600 
5601   // If function B doesn't have the "sign-return-address" attribute but A does,
5602   // the functions' signing behaviour is equal if A's value for
5603   // "sign-return-address" is "none" and vice versa.
5604   if (Fa.hasFnAttribute("sign-return-address")) {
5605     StringRef ScopeA =
5606         Fa.getFnAttribute("sign-return-address").getValueAsString();
5607     return ScopeA.equals("none");
5608   }
5609 
5610   if (Fb.hasFnAttribute("sign-return-address")) {
5611     StringRef ScopeB =
5612         Fb.getFnAttribute("sign-return-address").getValueAsString();
5613     return ScopeB.equals("none");
5614   }
5615 
5616   llvm_unreachable("Unkown combination of sign-return-address attributes");
5617 }
5618 
5619 static bool
5620 outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
5621                                        const outliner::Candidate &b) {
5622   const Function &Fa = a.getMF()->getFunction();
5623   const Function &Fb = b.getMF()->getFunction();
5624 
5625   // If none of the functions have the "sign-return-address-key" attribute
5626   // their keys are equal
5627   if (!Fa.hasFnAttribute("sign-return-address-key") &&
5628       !Fb.hasFnAttribute("sign-return-address-key")) {
5629     return true;
5630   }
5631 
5632   // If both functions have the "sign-return-address-key" attribute their
5633   // keys are equal if the values of "sign-return-address-key" are equal
5634   if (Fa.hasFnAttribute("sign-return-address-key") &&
5635       Fb.hasFnAttribute("sign-return-address-key")) {
5636     StringRef KeyA =
5637         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5638     StringRef KeyB =
5639         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5640     return KeyA.equals(KeyB);
5641   }
5642 
5643   // If B doesn't have the "sign-return-address-key" attribute, both keys are
5644   // equal, if function a has the default key (a_key)
5645   if (Fa.hasFnAttribute("sign-return-address-key")) {
5646     StringRef KeyA =
5647         Fa.getFnAttribute("sign-return-address-key").getValueAsString();
5648     return KeyA.equals_lower("a_key");
5649   }
5650 
5651   if (Fb.hasFnAttribute("sign-return-address-key")) {
5652     StringRef KeyB =
5653         Fb.getFnAttribute("sign-return-address-key").getValueAsString();
5654     return KeyB.equals_lower("a_key");
5655   }
5656 
5657   llvm_unreachable("Unkown combination of sign-return-address-key attributes");
5658 }
5659 
5660 static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
5661                                                 const outliner::Candidate &b) {
5662   const AArch64Subtarget &SubtargetA =
5663       a.getMF()->getSubtarget<AArch64Subtarget>();
5664   const AArch64Subtarget &SubtargetB =
5665       b.getMF()->getSubtarget<AArch64Subtarget>();
5666   return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
5667 }
5668 
5669 outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
5670     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5671   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5672   unsigned SequenceSize =
5673       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5674                       [this](unsigned Sum, const MachineInstr &MI) {
5675                         return Sum + getInstSizeInBytes(MI);
5676                       });
5677   unsigned NumBytesToCreateFrame = 0;
5678 
5679   // We only allow outlining for functions having exactly matching return
5680   // address signing attributes, i.e., all share the same value for the
5681   // attribute "sign-return-address" and all share the same type of key they
5682   // are signed with.
5683   // Additionally we require all functions to simultaniously either support
5684   // v8.3a features or not. Otherwise an outlined function could get signed
5685   // using dedicated v8.3 instructions and a call from a function that doesn't
5686   // support v8.3 instructions would therefore be invalid.
5687   if (std::adjacent_find(
5688           RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5689           [](const outliner::Candidate &a, const outliner::Candidate &b) {
5690             // Return true if a and b are non-equal w.r.t. return address
5691             // signing or support of v8.3a features
5692             if (outliningCandidatesSigningScopeConsensus(a, b) &&
5693                 outliningCandidatesSigningKeyConsensus(a, b) &&
5694                 outliningCandidatesV8_3OpsConsensus(a, b)) {
5695               return false;
5696             }
5697             return true;
5698           }) != RepeatedSequenceLocs.end()) {
5699     return outliner::OutlinedFunction();
5700   }
5701 
5702   // Since at this point all candidates agree on their return address signing
5703   // picking just one is fine. If the candidate functions potentially sign their
5704   // return addresses, the outlined function should do the same. Note that in
5705   // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5706   // not certainly true that the outlined function will have to sign its return
5707   // address but this decision is made later, when the decision to outline
5708   // has already been made.
5709   // The same holds for the number of additional instructions we need: On
5710   // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5711   // necessary. However, at this point we don't know if the outlined function
5712   // will have a RET instruction so we assume the worst.
5713   const Function &FCF = FirstCand.getMF()->getFunction();
5714   const TargetRegisterInfo &TRI = getRegisterInfo();
5715   if (FCF.hasFnAttribute("sign-return-address")) {
5716     // One PAC and one AUT instructions
5717     NumBytesToCreateFrame += 8;
5718 
5719     // We have to check if sp modifying instructions would get outlined.
5720     // If so we only allow outlining if sp is unchanged overall, so matching
5721     // sub and add instructions are okay to outline, all other sp modifications
5722     // are not
5723     auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5724       int SPValue = 0;
5725       MachineBasicBlock::iterator MBBI = C.front();
5726       for (;;) {
5727         if (MBBI->modifiesRegister(AArch64::SP, &TRI)) {
5728           switch (MBBI->getOpcode()) {
5729           case AArch64::ADDXri:
5730           case AArch64::ADDWri:
5731             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5732             assert(MBBI->getOperand(2).isImm() &&
5733                    "Expected operand to be immediate");
5734             assert(MBBI->getOperand(1).isReg() &&
5735                    "Expected operand to be a register");
5736             // Check if the add just increments sp. If so, we search for
5737             // matching sub instructions that decrement sp. If not, the
5738             // modification is illegal
5739             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5740               SPValue += MBBI->getOperand(2).getImm();
5741             else
5742               return true;
5743             break;
5744           case AArch64::SUBXri:
5745           case AArch64::SUBWri:
5746             assert(MBBI->getNumOperands() == 4 && "Wrong number of operands");
5747             assert(MBBI->getOperand(2).isImm() &&
5748                    "Expected operand to be immediate");
5749             assert(MBBI->getOperand(1).isReg() &&
5750                    "Expected operand to be a register");
5751             // Check if the sub just decrements sp. If so, we search for
5752             // matching add instructions that increment sp. If not, the
5753             // modification is illegal
5754             if (MBBI->getOperand(1).getReg() == AArch64::SP)
5755               SPValue -= MBBI->getOperand(2).getImm();
5756             else
5757               return true;
5758             break;
5759           default:
5760             return true;
5761           }
5762         }
5763         if (MBBI == C.back())
5764           break;
5765         ++MBBI;
5766       }
5767       if (SPValue)
5768         return true;
5769       return false;
5770     };
5771     // Remove candidates with illegal stack modifying instructions
5772     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5773                                               RepeatedSequenceLocs.end(),
5774                                               hasIllegalSPModification),
5775                                RepeatedSequenceLocs.end());
5776 
5777     // If the sequence doesn't have enough candidates left, then we're done.
5778     if (RepeatedSequenceLocs.size() < 2)
5779       return outliner::OutlinedFunction();
5780   }
5781 
5782   // Properties about candidate MBBs that hold for all of them.
5783   unsigned FlagsSetInAll = 0xF;
5784 
5785   // Compute liveness information for each candidate, and set FlagsSetInAll.
5786   std::for_each(RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5787                 [&FlagsSetInAll](outliner::Candidate &C) {
5788                   FlagsSetInAll &= C.Flags;
5789                 });
5790 
5791   // According to the AArch64 Procedure Call Standard, the following are
5792   // undefined on entry/exit from a function call:
5793   //
5794   // * Registers x16, x17, (and thus w16, w17)
5795   // * Condition codes (and thus the NZCV register)
5796   //
5797   // Because if this, we can't outline any sequence of instructions where
5798   // one
5799   // of these registers is live into/across it. Thus, we need to delete
5800   // those
5801   // candidates.
5802   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5803     // If the unsafe registers in this block are all dead, then we don't need
5804     // to compute liveness here.
5805     if (C.Flags & UnsafeRegsDead)
5806       return false;
5807     C.initLRU(TRI);
5808     LiveRegUnits LRU = C.LRU;
5809     return (!LRU.available(AArch64::W16) || !LRU.available(AArch64::W17) ||
5810             !LRU.available(AArch64::NZCV));
5811   };
5812 
5813   // Are there any candidates where those registers are live?
5814   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5815     // Erase every candidate that violates the restrictions above. (It could be
5816     // true that we have viable candidates, so it's not worth bailing out in
5817     // the case that, say, 1 out of 20 candidates violate the restructions.)
5818     RepeatedSequenceLocs.erase(std::remove_if(RepeatedSequenceLocs.begin(),
5819                                               RepeatedSequenceLocs.end(),
5820                                               CantGuaranteeValueAcrossCall),
5821                                RepeatedSequenceLocs.end());
5822 
5823     // If the sequence doesn't have enough candidates left, then we're done.
5824     if (RepeatedSequenceLocs.size() < 2)
5825       return outliner::OutlinedFunction();
5826   }
5827 
5828   // At this point, we have only "safe" candidates to outline. Figure out
5829   // frame + call instruction information.
5830 
5831   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5832 
5833   // Helper lambda which sets call information for every candidate.
5834   auto SetCandidateCallInfo =
5835       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5836         for (outliner::Candidate &C : RepeatedSequenceLocs)
5837           C.setCallInfo(CallID, NumBytesForCall);
5838       };
5839 
5840   unsigned FrameID = MachineOutlinerDefault;
5841   NumBytesToCreateFrame += 4;
5842 
5843   bool HasBTI = any_of(RepeatedSequenceLocs, [](outliner::Candidate &C) {
5844     return C.getMF()->getFunction().hasFnAttribute("branch-target-enforcement");
5845   });
5846 
5847   // Returns true if an instructions is safe to fix up, false otherwise.
5848   auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
5849     if (MI.isCall())
5850       return true;
5851 
5852     if (!MI.modifiesRegister(AArch64::SP, &TRI) &&
5853         !MI.readsRegister(AArch64::SP, &TRI))
5854       return true;
5855 
5856     // Any modification of SP will break our code to save/restore LR.
5857     // FIXME: We could handle some instructions which add a constant
5858     // offset to SP, with a bit more work.
5859     if (MI.modifiesRegister(AArch64::SP, &TRI))
5860       return false;
5861 
5862     // At this point, we have a stack instruction that we might need to
5863     // fix up. We'll handle it if it's a load or store.
5864     if (MI.mayLoadOrStore()) {
5865       const MachineOperand *Base; // Filled with the base operand of MI.
5866       int64_t Offset;             // Filled with the offset of MI.
5867 
5868       // Does it allow us to offset the base operand and is the base the
5869       // register SP?
5870       if (!getMemOperandWithOffset(MI, Base, Offset, &TRI) || !Base->isReg() ||
5871           Base->getReg() != AArch64::SP)
5872         return false;
5873 
5874       // Find the minimum/maximum offset for this instruction and check
5875       // if fixing it up would be in range.
5876       int64_t MinOffset,
5877           MaxOffset;  // Unscaled offsets for the instruction.
5878       unsigned Scale; // The scale to multiply the offsets by.
5879       unsigned DummyWidth;
5880       getMemOpInfo(MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
5881 
5882       Offset += 16; // Update the offset to what it would be if we outlined.
5883       if (Offset < MinOffset * Scale || Offset > MaxOffset * Scale)
5884         return false;
5885 
5886       // It's in range, so we can outline it.
5887       return true;
5888     }
5889 
5890     // FIXME: Add handling for instructions like "add x0, sp, #8".
5891 
5892     // We can't fix it up, so don't outline it.
5893     return false;
5894   };
5895 
5896   // True if it's possible to fix up each stack instruction in this sequence.
5897   // Important for frames/call variants that modify the stack.
5898   bool AllStackInstrsSafe = std::all_of(
5899       FirstCand.front(), std::next(FirstCand.back()), IsSafeToFixup);
5900 
5901   // If the last instruction in any candidate is a terminator, then we should
5902   // tail call all of the candidates.
5903   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
5904     FrameID = MachineOutlinerTailCall;
5905     NumBytesToCreateFrame = 0;
5906     SetCandidateCallInfo(MachineOutlinerTailCall, 4);
5907   }
5908 
5909   else if (LastInstrOpcode == AArch64::BL ||
5910            (LastInstrOpcode == AArch64::BLR && !HasBTI)) {
5911     // FIXME: Do we need to check if the code after this uses the value of LR?
5912     FrameID = MachineOutlinerThunk;
5913     NumBytesToCreateFrame = 0;
5914     SetCandidateCallInfo(MachineOutlinerThunk, 4);
5915   }
5916 
5917   else {
5918     // We need to decide how to emit calls + frames. We can always emit the same
5919     // frame if we don't need to save to the stack. If we have to save to the
5920     // stack, then we need a different frame.
5921     unsigned NumBytesNoStackCalls = 0;
5922     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
5923 
5924     // Check if we have to save LR.
5925     for (outliner::Candidate &C : RepeatedSequenceLocs) {
5926       C.initLRU(TRI);
5927 
5928       // If we have a noreturn caller, then we're going to be conservative and
5929       // say that we have to save LR. If we don't have a ret at the end of the
5930       // block, then we can't reason about liveness accurately.
5931       //
5932       // FIXME: We can probably do better than always disabling this in
5933       // noreturn functions by fixing up the liveness info.
5934       bool IsNoReturn =
5935           C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
5936 
5937       // Is LR available? If so, we don't need a save.
5938       if (C.LRU.available(AArch64::LR) && !IsNoReturn) {
5939         NumBytesNoStackCalls += 4;
5940         C.setCallInfo(MachineOutlinerNoLRSave, 4);
5941         CandidatesWithoutStackFixups.push_back(C);
5942       }
5943 
5944       // Is an unused register available? If so, we won't modify the stack, so
5945       // we can outline with the same frame type as those that don't save LR.
5946       else if (findRegisterToSaveLRTo(C)) {
5947         NumBytesNoStackCalls += 12;
5948         C.setCallInfo(MachineOutlinerRegSave, 12);
5949         CandidatesWithoutStackFixups.push_back(C);
5950       }
5951 
5952       // Is SP used in the sequence at all? If not, we don't have to modify
5953       // the stack, so we are guaranteed to get the same frame.
5954       else if (C.UsedInSequence.available(AArch64::SP)) {
5955         NumBytesNoStackCalls += 12;
5956         C.setCallInfo(MachineOutlinerDefault, 12);
5957         CandidatesWithoutStackFixups.push_back(C);
5958       }
5959 
5960       // If we outline this, we need to modify the stack. Pretend we don't
5961       // outline this by saving all of its bytes.
5962       else {
5963         NumBytesNoStackCalls += SequenceSize;
5964       }
5965     }
5966 
5967     // If there are no places where we have to save LR, then note that we
5968     // don't have to update the stack. Otherwise, give every candidate the
5969     // default call type, as long as it's safe to do so.
5970     if (!AllStackInstrsSafe ||
5971         NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
5972       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
5973       FrameID = MachineOutlinerNoLRSave;
5974     } else {
5975       SetCandidateCallInfo(MachineOutlinerDefault, 12);
5976     }
5977 
5978     // If we dropped all of the candidates, bail out here.
5979     if (RepeatedSequenceLocs.size() < 2) {
5980       RepeatedSequenceLocs.clear();
5981       return outliner::OutlinedFunction();
5982     }
5983   }
5984 
5985   // Does every candidate's MBB contain a call? If so, then we might have a call
5986   // in the range.
5987   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
5988     // Check if the range contains a call. These require a save + restore of the
5989     // link register.
5990     bool ModStackToSaveLR = false;
5991     if (std::any_of(FirstCand.front(), FirstCand.back(),
5992                     [](const MachineInstr &MI) { return MI.isCall(); }))
5993       ModStackToSaveLR = true;
5994 
5995     // Handle the last instruction separately. If this is a tail call, then the
5996     // last instruction is a call. We don't want to save + restore in this case.
5997     // However, it could be possible that the last instruction is a call without
5998     // it being valid to tail call this sequence. We should consider this as
5999     // well.
6000     else if (FrameID != MachineOutlinerThunk &&
6001              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6002       ModStackToSaveLR = true;
6003 
6004     if (ModStackToSaveLR) {
6005       // We can't fix up the stack. Bail out.
6006       if (!AllStackInstrsSafe) {
6007         RepeatedSequenceLocs.clear();
6008         return outliner::OutlinedFunction();
6009       }
6010 
6011       // Save + restore LR.
6012       NumBytesToCreateFrame += 8;
6013     }
6014   }
6015 
6016   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6017                                     NumBytesToCreateFrame, FrameID);
6018 }
6019 
6020 bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
6021     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6022   const Function &F = MF.getFunction();
6023 
6024   // Can F be deduplicated by the linker? If it can, don't outline from it.
6025   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6026     return false;
6027 
6028   // Don't outline from functions with section markings; the program could
6029   // expect that all the code is in the named section.
6030   // FIXME: Allow outlining from multiple functions with the same section
6031   // marking.
6032   if (F.hasSection())
6033     return false;
6034 
6035   // Outlining from functions with redzones is unsafe since the outliner may
6036   // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
6037   // outline from it.
6038   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
6039   if (!AFI || AFI->hasRedZone().getValueOr(true))
6040     return false;
6041 
6042   // It's safe to outline from MF.
6043   return true;
6044 }
6045 
6046 bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6047                                               unsigned &Flags) const {
6048   // Check if LR is available through all of the MBB. If it's not, then set
6049   // a flag.
6050   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6051          "Suitable Machine Function for outlining must track liveness");
6052   LiveRegUnits LRU(getRegisterInfo());
6053 
6054   std::for_each(MBB.rbegin(), MBB.rend(),
6055                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6056 
6057   // Check if each of the unsafe registers are available...
6058   bool W16AvailableInBlock = LRU.available(AArch64::W16);
6059   bool W17AvailableInBlock = LRU.available(AArch64::W17);
6060   bool NZCVAvailableInBlock = LRU.available(AArch64::NZCV);
6061 
6062   // If all of these are dead (and not live out), we know we don't have to check
6063   // them later.
6064   if (W16AvailableInBlock && W17AvailableInBlock && NZCVAvailableInBlock)
6065     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6066 
6067   // Now, add the live outs to the set.
6068   LRU.addLiveOuts(MBB);
6069 
6070   // If any of these registers is available in the MBB, but also a live out of
6071   // the block, then we know outlining is unsafe.
6072   if (W16AvailableInBlock && !LRU.available(AArch64::W16))
6073     return false;
6074   if (W17AvailableInBlock && !LRU.available(AArch64::W17))
6075     return false;
6076   if (NZCVAvailableInBlock && !LRU.available(AArch64::NZCV))
6077     return false;
6078 
6079   // Check if there's a call inside this MachineBasicBlock. If there is, then
6080   // set a flag.
6081   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6082     Flags |= MachineOutlinerMBBFlags::HasCalls;
6083 
6084   MachineFunction *MF = MBB.getParent();
6085 
6086   // In the event that we outline, we may have to save LR. If there is an
6087   // available register in the MBB, then we'll always save LR there. Check if
6088   // this is true.
6089   bool CanSaveLR = false;
6090   const AArch64RegisterInfo *ARI = static_cast<const AArch64RegisterInfo *>(
6091       MF->getSubtarget().getRegisterInfo());
6092 
6093   // Check if there is an available register across the sequence that we can
6094   // use.
6095   for (unsigned Reg : AArch64::GPR64RegClass) {
6096     if (!ARI->isReservedReg(*MF, Reg) && Reg != AArch64::LR &&
6097         Reg != AArch64::X16 && Reg != AArch64::X17 && LRU.available(Reg)) {
6098       CanSaveLR = true;
6099       break;
6100     }
6101   }
6102 
6103   // Check if we have a register we can save LR to, and if LR was used
6104   // somewhere. If both of those things are true, then we need to evaluate the
6105   // safety of outlining stack instructions later.
6106   if (!CanSaveLR && !LRU.available(AArch64::LR))
6107     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6108 
6109   return true;
6110 }
6111 
6112 outliner::InstrType
6113 AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6114                                    unsigned Flags) const {
6115   MachineInstr &MI = *MIT;
6116   MachineBasicBlock *MBB = MI.getParent();
6117   MachineFunction *MF = MBB->getParent();
6118   AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
6119 
6120   // Don't outline anything used for return address signing. The outlined
6121   // function will get signed later if needed
6122   switch (MI.getOpcode()) {
6123   case AArch64::PACIASP:
6124   case AArch64::PACIBSP:
6125   case AArch64::AUTIASP:
6126   case AArch64::AUTIBSP:
6127   case AArch64::RETAA:
6128   case AArch64::RETAB:
6129   case AArch64::EMITBKEY:
6130     return outliner::InstrType::Illegal;
6131   }
6132 
6133   // Don't outline LOHs.
6134   if (FuncInfo->getLOHRelated().count(&MI))
6135     return outliner::InstrType::Illegal;
6136 
6137   // Don't allow debug values to impact outlining type.
6138   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6139     return outliner::InstrType::Invisible;
6140 
6141   // At this point, KILL instructions don't really tell us much so we can go
6142   // ahead and skip over them.
6143   if (MI.isKill())
6144     return outliner::InstrType::Invisible;
6145 
6146   // Is this a terminator for a basic block?
6147   if (MI.isTerminator()) {
6148 
6149     // Is this the end of a function?
6150     if (MI.getParent()->succ_empty())
6151       return outliner::InstrType::Legal;
6152 
6153     // It's not, so don't outline it.
6154     return outliner::InstrType::Illegal;
6155   }
6156 
6157   // Make sure none of the operands are un-outlinable.
6158   for (const MachineOperand &MOP : MI.operands()) {
6159     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6160         MOP.isTargetIndex())
6161       return outliner::InstrType::Illegal;
6162 
6163     // If it uses LR or W30 explicitly, then don't touch it.
6164     if (MOP.isReg() && !MOP.isImplicit() &&
6165         (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
6166       return outliner::InstrType::Illegal;
6167   }
6168 
6169   // Special cases for instructions that can always be outlined, but will fail
6170   // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
6171   // be outlined because they don't require a *specific* value to be in LR.
6172   if (MI.getOpcode() == AArch64::ADRP)
6173     return outliner::InstrType::Legal;
6174 
6175   // If MI is a call we might be able to outline it. We don't want to outline
6176   // any calls that rely on the position of items on the stack. When we outline
6177   // something containing a call, we have to emit a save and restore of LR in
6178   // the outlined function. Currently, this always happens by saving LR to the
6179   // stack. Thus, if we outline, say, half the parameters for a function call
6180   // plus the call, then we'll break the callee's expectations for the layout
6181   // of the stack.
6182   //
6183   // FIXME: Allow calls to functions which construct a stack frame, as long
6184   // as they don't access arguments on the stack.
6185   // FIXME: Figure out some way to analyze functions defined in other modules.
6186   // We should be able to compute the memory usage based on the IR calling
6187   // convention, even if we can't see the definition.
6188   if (MI.isCall()) {
6189     // Get the function associated with the call. Look at each operand and find
6190     // the one that represents the callee and get its name.
6191     const Function *Callee = nullptr;
6192     for (const MachineOperand &MOP : MI.operands()) {
6193       if (MOP.isGlobal()) {
6194         Callee = dyn_cast<Function>(MOP.getGlobal());
6195         break;
6196       }
6197     }
6198 
6199     // Never outline calls to mcount.  There isn't any rule that would require
6200     // this, but the Linux kernel's "ftrace" feature depends on it.
6201     if (Callee && Callee->getName() == "\01_mcount")
6202       return outliner::InstrType::Illegal;
6203 
6204     // If we don't know anything about the callee, assume it depends on the
6205     // stack layout of the caller. In that case, it's only legal to outline
6206     // as a tail-call.  Whitelist the call instructions we know about so we
6207     // don't get unexpected results with call pseudo-instructions.
6208     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6209     if (MI.getOpcode() == AArch64::BLR || MI.getOpcode() == AArch64::BL)
6210       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6211 
6212     if (!Callee)
6213       return UnknownCallOutlineType;
6214 
6215     // We have a function we have information about. Check it if it's something
6216     // can safely outline.
6217     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6218 
6219     // We don't know what's going on with the callee at all. Don't touch it.
6220     if (!CalleeMF)
6221       return UnknownCallOutlineType;
6222 
6223     // Check if we know anything about the callee saves on the function. If we
6224     // don't, then don't touch it, since that implies that we haven't
6225     // computed anything about its stack frame yet.
6226     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6227     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6228         MFI.getNumObjects() > 0)
6229       return UnknownCallOutlineType;
6230 
6231     // At this point, we can say that CalleeMF ought to not pass anything on the
6232     // stack. Therefore, we can outline it.
6233     return outliner::InstrType::Legal;
6234   }
6235 
6236   // Don't outline positions.
6237   if (MI.isPosition())
6238     return outliner::InstrType::Illegal;
6239 
6240   // Don't touch the link register or W30.
6241   if (MI.readsRegister(AArch64::W30, &getRegisterInfo()) ||
6242       MI.modifiesRegister(AArch64::W30, &getRegisterInfo()))
6243     return outliner::InstrType::Illegal;
6244 
6245   // Don't outline BTI instructions, because that will prevent the outlining
6246   // site from being indirectly callable.
6247   if (MI.getOpcode() == AArch64::HINT) {
6248     int64_t Imm = MI.getOperand(0).getImm();
6249     if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
6250       return outliner::InstrType::Illegal;
6251   }
6252 
6253   return outliner::InstrType::Legal;
6254 }
6255 
6256 void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6257   for (MachineInstr &MI : MBB) {
6258     const MachineOperand *Base;
6259     unsigned Width;
6260     int64_t Offset;
6261 
6262     // Is this a load or store with an immediate offset with SP as the base?
6263     if (!MI.mayLoadOrStore() ||
6264         !getMemOperandWithOffsetWidth(MI, Base, Offset, Width, &RI) ||
6265         (Base->isReg() && Base->getReg() != AArch64::SP))
6266       continue;
6267 
6268     // It is, so we have to fix it up.
6269     unsigned Scale;
6270     int64_t Dummy1, Dummy2;
6271 
6272     MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(MI);
6273     assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
6274     getMemOpInfo(MI.getOpcode(), Scale, Width, Dummy1, Dummy2);
6275     assert(Scale != 0 && "Unexpected opcode!");
6276 
6277     // We've pushed the return address to the stack, so add 16 to the offset.
6278     // This is safe, since we already checked if it would overflow when we
6279     // checked if this instruction was legal to outline.
6280     int64_t NewImm = (Offset + 16) / Scale;
6281     StackOffsetOperand.setImm(NewImm);
6282   }
6283 }
6284 
6285 static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
6286                                  bool ShouldSignReturnAddr,
6287                                  bool ShouldSignReturnAddrWithAKey) {
6288   if (ShouldSignReturnAddr) {
6289     MachineBasicBlock::iterator MBBPAC = MBB.begin();
6290     MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator();
6291     const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
6292     const TargetInstrInfo *TII = Subtarget.getInstrInfo();
6293     DebugLoc DL;
6294 
6295     if (MBBAUT != MBB.end())
6296       DL = MBBAUT->getDebugLoc();
6297 
6298     // At the very beginning of the basic block we insert the following
6299     // depending on the key type
6300     //
6301     // a_key:                   b_key:
6302     //    PACIASP                   EMITBKEY
6303     //    CFI_INSTRUCTION           PACIBSP
6304     //                              CFI_INSTRUCTION
6305     if (ShouldSignReturnAddrWithAKey) {
6306       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIASP))
6307           .setMIFlag(MachineInstr::FrameSetup);
6308     } else {
6309       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::EMITBKEY))
6310           .setMIFlag(MachineInstr::FrameSetup);
6311       BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::PACIBSP))
6312           .setMIFlag(MachineInstr::FrameSetup);
6313     }
6314     unsigned CFIIndex =
6315         MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
6316     BuildMI(MBB, MBBPAC, DebugLoc(), TII->get(AArch64::CFI_INSTRUCTION))
6317         .addCFIIndex(CFIIndex)
6318         .setMIFlags(MachineInstr::FrameSetup);
6319 
6320     // If v8.3a features are available we can replace a RET instruction by
6321     // RETAA or RETAB and omit the AUT instructions
6322     if (Subtarget.hasV8_3aOps() && MBBAUT != MBB.end() &&
6323         MBBAUT->getOpcode() == AArch64::RET) {
6324       BuildMI(MBB, MBBAUT, DL,
6325               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6326                                                     : AArch64::RETAB))
6327           .copyImplicitOps(*MBBAUT);
6328       MBB.erase(MBBAUT);
6329     } else {
6330       BuildMI(MBB, MBBAUT, DL,
6331               TII->get(ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6332                                                     : AArch64::AUTIBSP))
6333           .setMIFlag(MachineInstr::FrameDestroy);
6334     }
6335   }
6336 }
6337 
6338 void AArch64InstrInfo::buildOutlinedFrame(
6339     MachineBasicBlock &MBB, MachineFunction &MF,
6340     const outliner::OutlinedFunction &OF) const {
6341   // For thunk outlining, rewrite the last instruction from a call to a
6342   // tail-call.
6343   if (OF.FrameConstructionID == MachineOutlinerThunk) {
6344     MachineInstr *Call = &*--MBB.instr_end();
6345     unsigned TailOpcode;
6346     if (Call->getOpcode() == AArch64::BL) {
6347       TailOpcode = AArch64::TCRETURNdi;
6348     } else {
6349       assert(Call->getOpcode() == AArch64::BLR);
6350       TailOpcode = AArch64::TCRETURNriALL;
6351     }
6352     MachineInstr *TC = BuildMI(MF, DebugLoc(), get(TailOpcode))
6353                            .add(Call->getOperand(0))
6354                            .addImm(0);
6355     MBB.insert(MBB.end(), TC);
6356     Call->eraseFromParent();
6357   }
6358 
6359   bool IsLeafFunction = true;
6360 
6361   // Is there a call in the outlined range?
6362   auto IsNonTailCall = [](const MachineInstr &MI) {
6363     return MI.isCall() && !MI.isReturn();
6364   };
6365 
6366   if (std::any_of(MBB.instr_begin(), MBB.instr_end(), IsNonTailCall)) {
6367     // Fix up the instructions in the range, since we're going to modify the
6368     // stack.
6369     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6370            "Can only fix up stack references once");
6371     fixupPostOutline(MBB);
6372 
6373     IsLeafFunction = false;
6374 
6375     // LR has to be a live in so that we can save it.
6376     MBB.addLiveIn(AArch64::LR);
6377 
6378     MachineBasicBlock::iterator It = MBB.begin();
6379     MachineBasicBlock::iterator Et = MBB.end();
6380 
6381     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6382         OF.FrameConstructionID == MachineOutlinerThunk)
6383       Et = std::prev(MBB.end());
6384 
6385     // Insert a save before the outlined region
6386     MachineInstr *STRXpre = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6387                                 .addReg(AArch64::SP, RegState::Define)
6388                                 .addReg(AArch64::LR)
6389                                 .addReg(AArch64::SP)
6390                                 .addImm(-16);
6391     It = MBB.insert(It, STRXpre);
6392 
6393     const TargetSubtargetInfo &STI = MF.getSubtarget();
6394     const MCRegisterInfo *MRI = STI.getRegisterInfo();
6395     unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true);
6396 
6397     // Add a CFI saying the stack was moved 16 B down.
6398     int64_t StackPosEntry =
6399         MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 16));
6400     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6401         .addCFIIndex(StackPosEntry)
6402         .setMIFlags(MachineInstr::FrameSetup);
6403 
6404     // Add a CFI saying that the LR that we want to find is now 16 B higher than
6405     // before.
6406     int64_t LRPosEntry =
6407         MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfReg, 16));
6408     BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
6409         .addCFIIndex(LRPosEntry)
6410         .setMIFlags(MachineInstr::FrameSetup);
6411 
6412     // Insert a restore before the terminator for the function.
6413     MachineInstr *LDRXpost = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6414                                  .addReg(AArch64::SP, RegState::Define)
6415                                  .addReg(AArch64::LR, RegState::Define)
6416                                  .addReg(AArch64::SP)
6417                                  .addImm(16);
6418     Et = MBB.insert(Et, LDRXpost);
6419   }
6420 
6421   // If a bunch of candidates reach this point they must agree on their return
6422   // address signing. It is therefore enough to just consider the signing
6423   // behaviour of one of them
6424   const Function &CF = OF.Candidates.front().getMF()->getFunction();
6425   bool ShouldSignReturnAddr = false;
6426   if (CF.hasFnAttribute("sign-return-address")) {
6427     StringRef Scope =
6428         CF.getFnAttribute("sign-return-address").getValueAsString();
6429     if (Scope.equals("all"))
6430       ShouldSignReturnAddr = true;
6431     else if (Scope.equals("non-leaf") && !IsLeafFunction)
6432       ShouldSignReturnAddr = true;
6433   }
6434 
6435   // a_key is the default
6436   bool ShouldSignReturnAddrWithAKey = true;
6437   if (CF.hasFnAttribute("sign-return-address-key")) {
6438     const StringRef Key =
6439         CF.getFnAttribute("sign-return-address-key").getValueAsString();
6440     // Key can either be a_key or b_key
6441     assert((Key.equals_lower("a_key") || Key.equals_lower("b_key")) &&
6442            "Return address signing key must be either a_key or b_key");
6443     ShouldSignReturnAddrWithAKey = Key.equals_lower("a_key");
6444   }
6445 
6446   // If this is a tail call outlined function, then there's already a return.
6447   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6448       OF.FrameConstructionID == MachineOutlinerThunk) {
6449     signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6450                          ShouldSignReturnAddrWithAKey);
6451     return;
6452   }
6453 
6454   // It's not a tail call, so we have to insert the return ourselves.
6455   MachineInstr *ret = BuildMI(MF, DebugLoc(), get(AArch64::RET))
6456                           .addReg(AArch64::LR, RegState::Undef);
6457   MBB.insert(MBB.end(), ret);
6458 
6459   signOutlinedFunction(MF, MBB, ShouldSignReturnAddr,
6460                        ShouldSignReturnAddrWithAKey);
6461 
6462   // Did we have to modify the stack by saving the link register?
6463   if (OF.FrameConstructionID != MachineOutlinerDefault)
6464     return;
6465 
6466   // We modified the stack.
6467   // Walk over the basic block and fix up all the stack accesses.
6468   fixupPostOutline(MBB);
6469 }
6470 
6471 MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
6472     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6473     MachineFunction &MF, const outliner::Candidate &C) const {
6474 
6475   // Are we tail calling?
6476   if (C.CallConstructionID == MachineOutlinerTailCall) {
6477     // If yes, then we can just branch to the label.
6478     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::TCRETURNdi))
6479                             .addGlobalAddress(M.getNamedValue(MF.getName()))
6480                             .addImm(0));
6481     return It;
6482   }
6483 
6484   // Are we saving the link register?
6485   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6486       C.CallConstructionID == MachineOutlinerThunk) {
6487     // No, so just insert the call.
6488     It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6489                             .addGlobalAddress(M.getNamedValue(MF.getName())));
6490     return It;
6491   }
6492 
6493   // We want to return the spot where we inserted the call.
6494   MachineBasicBlock::iterator CallPt;
6495 
6496   // Instructions for saving and restoring LR around the call instruction we're
6497   // going to insert.
6498   MachineInstr *Save;
6499   MachineInstr *Restore;
6500   // Can we save to a register?
6501   if (C.CallConstructionID == MachineOutlinerRegSave) {
6502     // FIXME: This logic should be sunk into a target-specific interface so that
6503     // we don't have to recompute the register.
6504     unsigned Reg = findRegisterToSaveLRTo(C);
6505     assert(Reg != 0 && "No callee-saved register available?");
6506 
6507     // Save and restore LR from that register.
6508     Save = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), Reg)
6509                .addReg(AArch64::XZR)
6510                .addReg(AArch64::LR)
6511                .addImm(0);
6512     Restore = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
6513                 .addReg(AArch64::XZR)
6514                 .addReg(Reg)
6515                 .addImm(0);
6516   } else {
6517     // We have the default case. Save and restore from SP.
6518     Save = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
6519                .addReg(AArch64::SP, RegState::Define)
6520                .addReg(AArch64::LR)
6521                .addReg(AArch64::SP)
6522                .addImm(-16);
6523     Restore = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
6524                   .addReg(AArch64::SP, RegState::Define)
6525                   .addReg(AArch64::LR, RegState::Define)
6526                   .addReg(AArch64::SP)
6527                   .addImm(16);
6528   }
6529 
6530   It = MBB.insert(It, Save);
6531   It++;
6532 
6533   // Insert the call.
6534   It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
6535                           .addGlobalAddress(M.getNamedValue(MF.getName())));
6536   CallPt = It;
6537   It++;
6538 
6539   It = MBB.insert(It, Restore);
6540   return CallPt;
6541 }
6542 
6543 bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
6544   MachineFunction &MF) const {
6545   return MF.getFunction().hasMinSize();
6546 }
6547 
6548 Optional<DestSourcePair>
6549 AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
6550 
6551   // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
6552   // and zero immediate operands used as an alias for mov instruction.
6553   if (MI.getOpcode() == AArch64::ORRWrs &&
6554       MI.getOperand(1).getReg() == AArch64::WZR &&
6555       MI.getOperand(3).getImm() == 0x0) {
6556     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6557   }
6558 
6559   if (MI.getOpcode() == AArch64::ORRXrs &&
6560       MI.getOperand(1).getReg() == AArch64::XZR &&
6561       MI.getOperand(3).getImm() == 0x0) {
6562     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
6563   }
6564 
6565   return None;
6566 }
6567 
6568 Optional<RegImmPair> AArch64InstrInfo::isAddImmediate(const MachineInstr &MI,
6569                                                       Register Reg) const {
6570   int Sign = 1;
6571   int64_t Offset = 0;
6572 
6573   // TODO: Handle cases where Reg is a super- or sub-register of the
6574   // destination register.
6575   if (Reg != MI.getOperand(0).getReg())
6576     return None;
6577 
6578   switch (MI.getOpcode()) {
6579   default:
6580     return None;
6581   case AArch64::SUBWri:
6582   case AArch64::SUBXri:
6583   case AArch64::SUBSWri:
6584   case AArch64::SUBSXri:
6585     Sign *= -1;
6586     LLVM_FALLTHROUGH;
6587   case AArch64::ADDSWri:
6588   case AArch64::ADDSXri:
6589   case AArch64::ADDWri:
6590   case AArch64::ADDXri: {
6591     // TODO: Third operand can be global address (usually some string).
6592     if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg() ||
6593         !MI.getOperand(2).isImm())
6594       return None;
6595     Offset = MI.getOperand(2).getImm() * Sign;
6596     int Shift = MI.getOperand(3).getImm();
6597     assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
6598     Offset = Offset << Shift;
6599   }
6600   }
6601   return RegImmPair{MI.getOperand(1).getReg(), Offset};
6602 }
6603 
6604 /// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
6605 /// the destination register then, if possible, describe the value in terms of
6606 /// the source register.
6607 static Optional<ParamLoadedValue>
6608 describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
6609                        const TargetInstrInfo *TII,
6610                        const TargetRegisterInfo *TRI) {
6611   auto DestSrc = TII->isCopyInstr(MI);
6612   if (!DestSrc)
6613     return None;
6614 
6615   Register DestReg = DestSrc->Destination->getReg();
6616   Register SrcReg = DestSrc->Source->getReg();
6617 
6618   auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
6619 
6620   // If the described register is the destination, just return the source.
6621   if (DestReg == DescribedReg)
6622     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6623 
6624   // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
6625   if (MI.getOpcode() == AArch64::ORRWrs &&
6626       TRI->isSuperRegister(DestReg, DescribedReg))
6627     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
6628 
6629   // We may need to describe the lower part of a ORRXrs move.
6630   if (MI.getOpcode() == AArch64::ORRXrs &&
6631       TRI->isSubRegister(DestReg, DescribedReg)) {
6632     Register SrcSubReg = TRI->getSubReg(SrcReg, AArch64::sub_32);
6633     return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr);
6634   }
6635 
6636   assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
6637          "Unhandled ORR[XW]rs copy case");
6638 
6639   return None;
6640 }
6641 
6642 Optional<ParamLoadedValue>
6643 AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
6644                                       Register Reg) const {
6645   const MachineFunction *MF = MI.getMF();
6646   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6647   switch (MI.getOpcode()) {
6648   case AArch64::MOVZWi:
6649   case AArch64::MOVZXi: {
6650     // MOVZWi may be used for producing zero-extended 32-bit immediates in
6651     // 64-bit parameters, so we need to consider super-registers.
6652     if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
6653       return None;
6654 
6655     if (!MI.getOperand(1).isImm())
6656       return None;
6657     int64_t Immediate = MI.getOperand(1).getImm();
6658     int Shift = MI.getOperand(2).getImm();
6659     return ParamLoadedValue(MachineOperand::CreateImm(Immediate << Shift),
6660                             nullptr);
6661   }
6662   case AArch64::ORRWrs:
6663   case AArch64::ORRXrs:
6664     return describeORRLoadedValue(MI, Reg, this, TRI);
6665   }
6666 
6667   return TargetInstrInfo::describeLoadedValue(MI, Reg);
6668 }
6669 
6670 #define GET_INSTRINFO_HELPERS
6671 #include "AArch64GenInstrInfo.inc"
6672