1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the AArch64 implementation of the TargetInstrInfo class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64InstrInfo.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64TargetMachine.h"
18 #include "MCTargetDesc/AArch64MCTargetDesc.h"
19 #include "Utils/AArch64BaseInfo.h"
20 #include "llvm/CodeGen/MachineConstantPool.h"
21 #include "llvm/CodeGen/MachineDominators.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunctionPass.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/IR/Function.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/TargetRegistry.h"
29 #include <algorithm>
30 
31 #define GET_INSTRINFO_CTOR_DTOR
32 #include "AArch64GenInstrInfo.inc"
33 
34 using namespace llvm;
35 
36 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
37   : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP),
38     Subtarget(STI) {}
39 
40 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
41                                    MachineBasicBlock::iterator I, DebugLoc DL,
42                                    unsigned DestReg, unsigned SrcReg,
43                                    bool KillSrc) const {
44   unsigned Opc = 0;
45   unsigned ZeroReg = 0;
46   if (DestReg == AArch64::XSP || SrcReg == AArch64::XSP) {
47     // E.g. ADD xDst, xsp, #0 (, lsl #0)
48     BuildMI(MBB, I, DL, get(AArch64::ADDxxi_lsl0_s), DestReg)
49       .addReg(SrcReg)
50       .addImm(0);
51     return;
52   } else if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
53     // E.g. ADD wDST, wsp, #0 (, lsl #0)
54     BuildMI(MBB, I, DL, get(AArch64::ADDwwi_lsl0_s), DestReg)
55       .addReg(SrcReg)
56       .addImm(0);
57     return;
58   } else if (DestReg == AArch64::NZCV) {
59     assert(AArch64::GPR64RegClass.contains(SrcReg));
60     // E.g. MSR NZCV, xDST
61     BuildMI(MBB, I, DL, get(AArch64::MSRix))
62       .addImm(A64SysReg::NZCV)
63       .addReg(SrcReg);
64   } else if (SrcReg == AArch64::NZCV) {
65     assert(AArch64::GPR64RegClass.contains(DestReg));
66     // E.g. MRS xDST, NZCV
67     BuildMI(MBB, I, DL, get(AArch64::MRSxi), DestReg)
68       .addImm(A64SysReg::NZCV);
69   } else if (AArch64::GPR64RegClass.contains(DestReg)) {
70     if(AArch64::GPR64RegClass.contains(SrcReg)){
71       Opc = AArch64::ORRxxx_lsl;
72       ZeroReg = AArch64::XZR;
73     } else{
74       assert(AArch64::FPR64RegClass.contains(SrcReg));
75       BuildMI(MBB, I, DL, get(AArch64::FMOVxd), DestReg)
76         .addReg(SrcReg);
77       return;
78     }
79   } else if (AArch64::GPR32RegClass.contains(DestReg)) {
80     if(AArch64::GPR32RegClass.contains(SrcReg)){
81       Opc = AArch64::ORRwww_lsl;
82       ZeroReg = AArch64::WZR;
83     } else{
84       assert(AArch64::FPR32RegClass.contains(SrcReg));
85       BuildMI(MBB, I, DL, get(AArch64::FMOVws), DestReg)
86         .addReg(SrcReg);
87       return;
88     }
89   } else if (AArch64::FPR32RegClass.contains(DestReg)) {
90     if(AArch64::FPR32RegClass.contains(SrcReg)){
91       BuildMI(MBB, I, DL, get(AArch64::FMOVss), DestReg)
92         .addReg(SrcReg);
93       return;
94     }
95     else {
96       assert(AArch64::GPR32RegClass.contains(SrcReg));
97       BuildMI(MBB, I, DL, get(AArch64::FMOVsw), DestReg)
98         .addReg(SrcReg);
99       return;
100     }
101   } else if (AArch64::FPR64RegClass.contains(DestReg)) {
102     if(AArch64::FPR64RegClass.contains(SrcReg)){
103       BuildMI(MBB, I, DL, get(AArch64::FMOVdd), DestReg)
104         .addReg(SrcReg);
105       return;
106     }
107     else {
108       assert(AArch64::GPR64RegClass.contains(SrcReg));
109       BuildMI(MBB, I, DL, get(AArch64::FMOVdx), DestReg)
110         .addReg(SrcReg);
111       return;
112     }
113   } else if (AArch64::FPR128RegClass.contains(DestReg)) {
114     assert(AArch64::FPR128RegClass.contains(SrcReg));
115 
116     // If NEON is enable, we use ORR to implement this copy.
117     // If NEON isn't available, emit STR and LDR to handle this.
118     if(getSubTarget().hasNEON()) {
119       BuildMI(MBB, I, DL, get(AArch64::ORRvvv_16B), DestReg)
120         .addReg(SrcReg)
121         .addReg(SrcReg);
122       return;
123     } else {
124       BuildMI(MBB, I, DL, get(AArch64::LSFP128_PreInd_STR), AArch64::XSP)
125         .addReg(SrcReg)
126         .addReg(AArch64::XSP)
127         .addImm(0x1ff & -16);
128 
129       BuildMI(MBB, I, DL, get(AArch64::LSFP128_PostInd_LDR), DestReg)
130         .addReg(AArch64::XSP, RegState::Define)
131         .addReg(AArch64::XSP)
132         .addImm(16);
133       return;
134     }
135   } else if (AArch64::FPR16RegClass.contains(DestReg, SrcReg)) {
136     // The copy of two FPR16 registers is implemented by the copy of two FPR32
137     const TargetRegisterInfo *TRI = &getRegisterInfo();
138     unsigned Dst = TRI->getMatchingSuperReg(DestReg, AArch64::sub_16,
139                                             &AArch64::FPR32RegClass);
140     unsigned Src = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_16,
141                                             &AArch64::FPR32RegClass);
142     BuildMI(MBB, I, DL, get(AArch64::FMOVss), Dst)
143       .addReg(Src);
144     return;
145   } else {
146     CopyPhysRegTuple(MBB, I, DL, DestReg, SrcReg);
147     return;
148   }
149 
150   // E.g. ORR xDst, xzr, xSrc, lsl #0
151   BuildMI(MBB, I, DL, get(Opc), DestReg)
152     .addReg(ZeroReg)
153     .addReg(SrcReg)
154     .addImm(0);
155 }
156 
157 void AArch64InstrInfo::CopyPhysRegTuple(MachineBasicBlock &MBB,
158                                         MachineBasicBlock::iterator I,
159                                         DebugLoc DL, unsigned DestReg,
160                                         unsigned SrcReg) const {
161   unsigned SubRegs;
162   bool IsQRegs;
163   if (AArch64::DPairRegClass.contains(DestReg, SrcReg)) {
164     SubRegs = 2;
165     IsQRegs = false;
166   } else if (AArch64::DTripleRegClass.contains(DestReg, SrcReg)) {
167     SubRegs = 3;
168     IsQRegs = false;
169   } else if (AArch64::DQuadRegClass.contains(DestReg, SrcReg)) {
170     SubRegs = 4;
171     IsQRegs = false;
172   } else if (AArch64::QPairRegClass.contains(DestReg, SrcReg)) {
173     SubRegs = 2;
174     IsQRegs = true;
175   } else if (AArch64::QTripleRegClass.contains(DestReg, SrcReg)) {
176     SubRegs = 3;
177     IsQRegs = true;
178   } else if (AArch64::QQuadRegClass.contains(DestReg, SrcReg)) {
179     SubRegs = 4;
180     IsQRegs = true;
181   } else
182     llvm_unreachable("Unknown register class");
183 
184   unsigned BeginIdx = IsQRegs ? AArch64::qsub_0 : AArch64::dsub_0;
185   int Spacing = 1;
186   const TargetRegisterInfo *TRI = &getRegisterInfo();
187   // Copy register tuples backward when the first Dest reg overlaps
188   // with SrcReg.
189   if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
190     BeginIdx = BeginIdx + (SubRegs - 1);
191     Spacing = -1;
192   }
193 
194   unsigned Opc = IsQRegs ? AArch64::ORRvvv_16B : AArch64::ORRvvv_8B;
195   for (unsigned i = 0; i != SubRegs; ++i) {
196     unsigned Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
197     unsigned Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
198     assert(Dst && Src && "Bad sub-register");
199     BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst)
200         .addReg(Src)
201         .addReg(Src);
202   }
203   return;
204 }
205 
206 /// Does the Opcode represent a conditional branch that we can remove and re-add
207 /// at the end of a basic block?
208 static bool isCondBranch(unsigned Opc) {
209   return Opc == AArch64::Bcc || Opc == AArch64::CBZw || Opc == AArch64::CBZx ||
210          Opc == AArch64::CBNZw || Opc == AArch64::CBNZx ||
211          Opc == AArch64::TBZwii || Opc == AArch64::TBZxii ||
212          Opc == AArch64::TBNZwii || Opc == AArch64::TBNZxii;
213 }
214 
215 /// Takes apart a given conditional branch MachineInstr (see isCondBranch),
216 /// setting TBB to the destination basic block and populating the Cond vector
217 /// with data necessary to recreate the conditional branch at a later
218 /// date. First element will be the opcode, and subsequent ones define the
219 /// conditions being branched on in an instruction-specific manner.
220 static void classifyCondBranch(MachineInstr *I, MachineBasicBlock *&TBB,
221                                SmallVectorImpl<MachineOperand> &Cond) {
222   switch(I->getOpcode()) {
223   case AArch64::Bcc:
224   case AArch64::CBZw:
225   case AArch64::CBZx:
226   case AArch64::CBNZw:
227   case AArch64::CBNZx:
228     // These instructions just have one predicate operand in position 0 (either
229     // a condition code or a register being compared).
230     Cond.push_back(MachineOperand::CreateImm(I->getOpcode()));
231     Cond.push_back(I->getOperand(0));
232     TBB = I->getOperand(1).getMBB();
233     return;
234   case AArch64::TBZwii:
235   case AArch64::TBZxii:
236   case AArch64::TBNZwii:
237   case AArch64::TBNZxii:
238     // These have two predicate operands: a register and a bit position.
239     Cond.push_back(MachineOperand::CreateImm(I->getOpcode()));
240     Cond.push_back(I->getOperand(0));
241     Cond.push_back(I->getOperand(1));
242     TBB = I->getOperand(2).getMBB();
243     return;
244   default:
245     llvm_unreachable("Unknown conditional branch to classify");
246   }
247 }
248 
249 
250 bool
251 AArch64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
252                                 MachineBasicBlock *&FBB,
253                                 SmallVectorImpl<MachineOperand> &Cond,
254                                 bool AllowModify) const {
255   // If the block has no terminators, it just falls into the block after it.
256   MachineBasicBlock::iterator I = MBB.end();
257   if (I == MBB.begin())
258     return false;
259   --I;
260   while (I->isDebugValue()) {
261     if (I == MBB.begin())
262       return false;
263     --I;
264   }
265   if (!isUnpredicatedTerminator(I))
266     return false;
267 
268   // Get the last instruction in the block.
269   MachineInstr *LastInst = I;
270 
271   // If there is only one terminator instruction, process it.
272   unsigned LastOpc = LastInst->getOpcode();
273   if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
274     if (LastOpc == AArch64::Bimm) {
275       TBB = LastInst->getOperand(0).getMBB();
276       return false;
277     }
278     if (isCondBranch(LastOpc)) {
279       classifyCondBranch(LastInst, TBB, Cond);
280       return false;
281     }
282     return true;  // Can't handle indirect branch.
283   }
284 
285   // Get the instruction before it if it is a terminator.
286   MachineInstr *SecondLastInst = I;
287   unsigned SecondLastOpc = SecondLastInst->getOpcode();
288 
289   // If AllowModify is true and the block ends with two or more unconditional
290   // branches, delete all but the first unconditional branch.
291   if (AllowModify && LastOpc == AArch64::Bimm) {
292     while (SecondLastOpc == AArch64::Bimm) {
293       LastInst->eraseFromParent();
294       LastInst = SecondLastInst;
295       LastOpc = LastInst->getOpcode();
296       if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
297         // Return now the only terminator is an unconditional branch.
298         TBB = LastInst->getOperand(0).getMBB();
299         return false;
300       } else {
301         SecondLastInst = I;
302         SecondLastOpc = SecondLastInst->getOpcode();
303       }
304     }
305   }
306 
307   // If there are three terminators, we don't know what sort of block this is.
308   if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I))
309     return true;
310 
311   // If the block ends with a B and a Bcc, handle it.
312   if (LastOpc == AArch64::Bimm) {
313     if (SecondLastOpc == AArch64::Bcc) {
314       TBB =  SecondLastInst->getOperand(1).getMBB();
315       Cond.push_back(MachineOperand::CreateImm(AArch64::Bcc));
316       Cond.push_back(SecondLastInst->getOperand(0));
317       FBB = LastInst->getOperand(0).getMBB();
318       return false;
319     } else if (isCondBranch(SecondLastOpc)) {
320       classifyCondBranch(SecondLastInst, TBB, Cond);
321       FBB = LastInst->getOperand(0).getMBB();
322       return false;
323     }
324   }
325 
326   // If the block ends with two unconditional branches, handle it.  The second
327   // one is not executed, so remove it.
328   if (SecondLastOpc == AArch64::Bimm && LastOpc == AArch64::Bimm) {
329     TBB = SecondLastInst->getOperand(0).getMBB();
330     I = LastInst;
331     if (AllowModify)
332       I->eraseFromParent();
333     return false;
334   }
335 
336   // Otherwise, can't handle this.
337   return true;
338 }
339 
340 bool AArch64InstrInfo::ReverseBranchCondition(
341                                   SmallVectorImpl<MachineOperand> &Cond) const {
342   switch (Cond[0].getImm()) {
343   case AArch64::Bcc: {
344     A64CC::CondCodes CC = static_cast<A64CC::CondCodes>(Cond[1].getImm());
345     CC = A64InvertCondCode(CC);
346     Cond[1].setImm(CC);
347     return false;
348   }
349   case AArch64::CBZw:
350     Cond[0].setImm(AArch64::CBNZw);
351     return false;
352   case AArch64::CBZx:
353     Cond[0].setImm(AArch64::CBNZx);
354     return false;
355   case AArch64::CBNZw:
356     Cond[0].setImm(AArch64::CBZw);
357     return false;
358   case AArch64::CBNZx:
359     Cond[0].setImm(AArch64::CBZx);
360     return false;
361   case AArch64::TBZwii:
362     Cond[0].setImm(AArch64::TBNZwii);
363     return false;
364   case AArch64::TBZxii:
365     Cond[0].setImm(AArch64::TBNZxii);
366     return false;
367   case AArch64::TBNZwii:
368     Cond[0].setImm(AArch64::TBZwii);
369     return false;
370   case AArch64::TBNZxii:
371     Cond[0].setImm(AArch64::TBZxii);
372     return false;
373   default:
374     llvm_unreachable("Unknown branch type");
375   }
376 }
377 
378 
379 unsigned
380 AArch64InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
381                                MachineBasicBlock *FBB,
382                                const SmallVectorImpl<MachineOperand> &Cond,
383                                DebugLoc DL) const {
384   if (FBB == 0 && Cond.empty()) {
385     BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(TBB);
386     return 1;
387   } else if (FBB == 0) {
388     MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
389     for (int i = 1, e = Cond.size(); i != e; ++i)
390       MIB.addOperand(Cond[i]);
391     MIB.addMBB(TBB);
392     return 1;
393   }
394 
395   MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
396   for (int i = 1, e = Cond.size(); i != e; ++i)
397     MIB.addOperand(Cond[i]);
398   MIB.addMBB(TBB);
399 
400   BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(FBB);
401   return 2;
402 }
403 
404 unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
405   MachineBasicBlock::iterator I = MBB.end();
406   if (I == MBB.begin()) return 0;
407   --I;
408   while (I->isDebugValue()) {
409     if (I == MBB.begin())
410       return 0;
411     --I;
412   }
413   if (I->getOpcode() != AArch64::Bimm && !isCondBranch(I->getOpcode()))
414     return 0;
415 
416   // Remove the branch.
417   I->eraseFromParent();
418 
419   I = MBB.end();
420 
421   if (I == MBB.begin()) return 1;
422   --I;
423   if (!isCondBranch(I->getOpcode()))
424     return 1;
425 
426   // Remove the branch.
427   I->eraseFromParent();
428   return 2;
429 }
430 
431 bool
432 AArch64InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MBBI) const {
433   MachineInstr &MI = *MBBI;
434   MachineBasicBlock &MBB = *MI.getParent();
435 
436   unsigned Opcode = MI.getOpcode();
437   switch (Opcode) {
438   case AArch64::TLSDESC_BLRx: {
439     MachineInstr *NewMI =
440       BuildMI(MBB, MBBI, MI.getDebugLoc(), get(AArch64::TLSDESCCALL))
441         .addOperand(MI.getOperand(1));
442     MI.setDesc(get(AArch64::BLRx));
443 
444     llvm::finalizeBundle(MBB, NewMI, *++MBBI);
445     return true;
446     }
447   default:
448     return false;
449   }
450 
451   return false;
452 }
453 
454 void
455 AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
456                                       MachineBasicBlock::iterator MBBI,
457                                       unsigned SrcReg, bool isKill,
458                                       int FrameIdx,
459                                       const TargetRegisterClass *RC,
460                                       const TargetRegisterInfo *TRI) const {
461   DebugLoc DL = MBB.findDebugLoc(MBBI);
462   MachineFunction &MF = *MBB.getParent();
463   MachineFrameInfo &MFI = *MF.getFrameInfo();
464   unsigned Align = MFI.getObjectAlignment(FrameIdx);
465 
466   MachineMemOperand *MMO
467     = MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FrameIdx),
468                               MachineMemOperand::MOStore,
469                               MFI.getObjectSize(FrameIdx),
470                               Align);
471 
472   unsigned StoreOp = 0;
473   if (RC->hasType(MVT::i64) || RC->hasType(MVT::i32)) {
474     switch(RC->getSize()) {
475     case 4: StoreOp = AArch64::LS32_STR; break;
476     case 8: StoreOp = AArch64::LS64_STR; break;
477     default:
478       llvm_unreachable("Unknown size for regclass");
479     }
480   } else if (RC->hasType(MVT::f32) || RC->hasType(MVT::f64) ||
481              RC->hasType(MVT::f128)) {
482     switch (RC->getSize()) {
483     case 4: StoreOp = AArch64::LSFP32_STR; break;
484     case 8: StoreOp = AArch64::LSFP64_STR; break;
485     case 16: StoreOp = AArch64::LSFP128_STR; break;
486     default:
487       llvm_unreachable("Unknown size for regclass");
488     }
489   } else { // For a super register class has more than one sub registers
490     if (AArch64::DPairRegClass.hasSubClassEq(RC))
491       StoreOp = AArch64::ST1x2_8B;
492     else if (AArch64::DTripleRegClass.hasSubClassEq(RC))
493       StoreOp = AArch64::ST1x3_8B;
494     else if (AArch64::DQuadRegClass.hasSubClassEq(RC))
495       StoreOp = AArch64::ST1x4_8B;
496     else if (AArch64::QPairRegClass.hasSubClassEq(RC))
497       StoreOp = AArch64::ST1x2_16B;
498     else if (AArch64::QTripleRegClass.hasSubClassEq(RC))
499       StoreOp = AArch64::ST1x3_16B;
500     else if (AArch64::QQuadRegClass.hasSubClassEq(RC))
501       StoreOp = AArch64::ST1x4_16B;
502     else
503       llvm_unreachable("Unknown reg class");
504 
505     MachineInstrBuilder NewMI = BuildMI(MBB, MBBI, DL, get(StoreOp));
506     // Vector store has different operands from other store instructions.
507     NewMI.addFrameIndex(FrameIdx)
508          .addReg(SrcReg, getKillRegState(isKill))
509          .addMemOperand(MMO);
510     return;
511   }
512 
513   MachineInstrBuilder NewMI = BuildMI(MBB, MBBI, DL, get(StoreOp));
514   NewMI.addReg(SrcReg, getKillRegState(isKill))
515     .addFrameIndex(FrameIdx)
516     .addImm(0)
517     .addMemOperand(MMO);
518 
519 }
520 
521 void
522 AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
523                                        MachineBasicBlock::iterator MBBI,
524                                        unsigned DestReg, int FrameIdx,
525                                        const TargetRegisterClass *RC,
526                                        const TargetRegisterInfo *TRI) const {
527   DebugLoc DL = MBB.findDebugLoc(MBBI);
528   MachineFunction &MF = *MBB.getParent();
529   MachineFrameInfo &MFI = *MF.getFrameInfo();
530   unsigned Align = MFI.getObjectAlignment(FrameIdx);
531 
532   MachineMemOperand *MMO
533     = MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FrameIdx),
534                               MachineMemOperand::MOLoad,
535                               MFI.getObjectSize(FrameIdx),
536                               Align);
537 
538   unsigned LoadOp = 0;
539   if (RC->hasType(MVT::i64) || RC->hasType(MVT::i32)) {
540     switch(RC->getSize()) {
541     case 4: LoadOp = AArch64::LS32_LDR; break;
542     case 8: LoadOp = AArch64::LS64_LDR; break;
543     default:
544       llvm_unreachable("Unknown size for regclass");
545     }
546   } else if (RC->hasType(MVT::f32) || RC->hasType(MVT::f64) ||
547              RC->hasType(MVT::f128)) {
548     switch (RC->getSize()) {
549     case 4: LoadOp = AArch64::LSFP32_LDR; break;
550     case 8: LoadOp = AArch64::LSFP64_LDR; break;
551     case 16: LoadOp = AArch64::LSFP128_LDR; break;
552     default:
553       llvm_unreachable("Unknown size for regclass");
554     }
555   } else { // For a super register class has more than one sub registers
556     if (AArch64::DPairRegClass.hasSubClassEq(RC))
557       LoadOp = AArch64::LD1x2_8B;
558     else if (AArch64::DTripleRegClass.hasSubClassEq(RC))
559       LoadOp = AArch64::LD1x3_8B;
560     else if (AArch64::DQuadRegClass.hasSubClassEq(RC))
561       LoadOp = AArch64::LD1x4_8B;
562     else if (AArch64::QPairRegClass.hasSubClassEq(RC))
563       LoadOp = AArch64::LD1x2_16B;
564     else if (AArch64::QTripleRegClass.hasSubClassEq(RC))
565       LoadOp = AArch64::LD1x3_16B;
566     else if (AArch64::QQuadRegClass.hasSubClassEq(RC))
567       LoadOp = AArch64::LD1x4_16B;
568     else
569       llvm_unreachable("Unknown reg class");
570 
571     MachineInstrBuilder NewMI = BuildMI(MBB, MBBI, DL, get(LoadOp), DestReg);
572     // Vector load has different operands from other load instructions.
573     NewMI.addFrameIndex(FrameIdx)
574          .addMemOperand(MMO);
575     return;
576   }
577 
578   MachineInstrBuilder NewMI = BuildMI(MBB, MBBI, DL, get(LoadOp), DestReg);
579   NewMI.addFrameIndex(FrameIdx)
580        .addImm(0)
581        .addMemOperand(MMO);
582 }
583 
584 unsigned AArch64InstrInfo::estimateRSStackLimit(MachineFunction &MF) const {
585   unsigned Limit = (1 << 16) - 1;
586   for (MachineFunction::iterator BB = MF.begin(),E = MF.end(); BB != E; ++BB) {
587     for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end();
588          I != E; ++I) {
589       for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
590         if (!I->getOperand(i).isFI()) continue;
591 
592         // When using ADDxxi_lsl0_s to get the address of a stack object, 0xfff
593         // is the largest offset guaranteed to fit in the immediate offset.
594         if (I->getOpcode() == AArch64::ADDxxi_lsl0_s) {
595           Limit = std::min(Limit, 0xfffu);
596           break;
597         }
598 
599         int AccessScale, MinOffset, MaxOffset;
600         getAddressConstraints(*I, AccessScale, MinOffset, MaxOffset);
601         Limit = std::min(Limit, static_cast<unsigned>(MaxOffset));
602 
603         break; // At most one FI per instruction
604       }
605     }
606   }
607 
608   return Limit;
609 }
610 void AArch64InstrInfo::getAddressConstraints(const MachineInstr &MI,
611                                              int &AccessScale, int &MinOffset,
612                                              int &MaxOffset) const {
613   switch (MI.getOpcode()) {
614   default:
615     llvm_unreachable("Unknown load/store kind");
616   case TargetOpcode::DBG_VALUE:
617     AccessScale = 1;
618     MinOffset = INT_MIN;
619     MaxOffset = INT_MAX;
620     return;
621   case AArch64::LS8_LDR: case AArch64::LS8_STR:
622   case AArch64::LSFP8_LDR: case AArch64::LSFP8_STR:
623   case AArch64::LDRSBw:
624   case AArch64::LDRSBx:
625     AccessScale = 1;
626     MinOffset = 0;
627     MaxOffset = 0xfff;
628     return;
629   case AArch64::LS16_LDR: case AArch64::LS16_STR:
630   case AArch64::LSFP16_LDR: case AArch64::LSFP16_STR:
631   case AArch64::LDRSHw:
632   case AArch64::LDRSHx:
633     AccessScale = 2;
634     MinOffset = 0;
635     MaxOffset = 0xfff * AccessScale;
636     return;
637   case AArch64::LS32_LDR:  case AArch64::LS32_STR:
638   case AArch64::LSFP32_LDR: case AArch64::LSFP32_STR:
639   case AArch64::LDRSWx:
640   case AArch64::LDPSWx:
641     AccessScale = 4;
642     MinOffset = 0;
643     MaxOffset = 0xfff * AccessScale;
644     return;
645   case AArch64::LS64_LDR: case AArch64::LS64_STR:
646   case AArch64::LSFP64_LDR: case AArch64::LSFP64_STR:
647   case AArch64::PRFM:
648     AccessScale = 8;
649     MinOffset = 0;
650     MaxOffset = 0xfff * AccessScale;
651     return;
652   case AArch64::LSFP128_LDR: case AArch64::LSFP128_STR:
653     AccessScale = 16;
654     MinOffset = 0;
655     MaxOffset = 0xfff * AccessScale;
656     return;
657   case AArch64::LSPair32_LDR: case AArch64::LSPair32_STR:
658   case AArch64::LSFPPair32_LDR: case AArch64::LSFPPair32_STR:
659     AccessScale = 4;
660     MinOffset = -0x40 * AccessScale;
661     MaxOffset = 0x3f * AccessScale;
662     return;
663   case AArch64::LSPair64_LDR: case AArch64::LSPair64_STR:
664   case AArch64::LSFPPair64_LDR: case AArch64::LSFPPair64_STR:
665     AccessScale = 8;
666     MinOffset = -0x40 * AccessScale;
667     MaxOffset = 0x3f * AccessScale;
668     return;
669   case AArch64::LSFPPair128_LDR: case AArch64::LSFPPair128_STR:
670     AccessScale = 16;
671     MinOffset = -0x40 * AccessScale;
672     MaxOffset = 0x3f * AccessScale;
673     return;
674   case AArch64::LD1x2_8B: case AArch64::ST1x2_8B:
675     AccessScale = 16;
676     MinOffset = 0;
677     MaxOffset = 0xfff * AccessScale;
678     return;
679   case AArch64::LD1x3_8B: case AArch64::ST1x3_8B:
680     AccessScale = 24;
681     MinOffset = 0;
682     MaxOffset = 0xfff * AccessScale;
683     return;
684   case AArch64::LD1x4_8B: case AArch64::ST1x4_8B:
685   case AArch64::LD1x2_16B: case AArch64::ST1x2_16B:
686     AccessScale = 32;
687     MinOffset = 0;
688     MaxOffset = 0xfff * AccessScale;
689     return;
690   case AArch64::LD1x3_16B: case AArch64::ST1x3_16B:
691     AccessScale = 48;
692     MinOffset = 0;
693     MaxOffset = 0xfff * AccessScale;
694     return;
695   case AArch64::LD1x4_16B: case AArch64::ST1x4_16B:
696     AccessScale = 64;
697     MinOffset = 0;
698     MaxOffset = 0xfff * AccessScale;
699     return;
700   }
701 }
702 
703 unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
704   const MCInstrDesc &MCID = MI.getDesc();
705   const MachineBasicBlock &MBB = *MI.getParent();
706   const MachineFunction &MF = *MBB.getParent();
707   const MCAsmInfo &MAI = *MF.getTarget().getMCAsmInfo();
708 
709   if (MCID.getSize())
710     return MCID.getSize();
711 
712   if (MI.getOpcode() == AArch64::INLINEASM)
713     return getInlineAsmLength(MI.getOperand(0).getSymbolName(), MAI);
714 
715   if (MI.isLabel())
716     return 0;
717 
718   switch (MI.getOpcode()) {
719   case TargetOpcode::BUNDLE:
720     return getInstBundleLength(MI);
721   case TargetOpcode::IMPLICIT_DEF:
722   case TargetOpcode::KILL:
723   case TargetOpcode::PROLOG_LABEL:
724   case TargetOpcode::EH_LABEL:
725   case TargetOpcode::DBG_VALUE:
726     return 0;
727   case AArch64::TLSDESCCALL:
728     return 0;
729   default:
730     llvm_unreachable("Unknown instruction class");
731   }
732 }
733 
734 unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
735   unsigned Size = 0;
736   MachineBasicBlock::const_instr_iterator I = MI;
737   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
738   while (++I != E && I->isInsideBundle()) {
739     assert(!I->isBundle() && "No nested bundle!");
740     Size += getInstSizeInBytes(*I);
741   }
742   return Size;
743 }
744 
745 bool llvm::rewriteA64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
746                                 unsigned FrameReg, int &Offset,
747                                 const AArch64InstrInfo &TII) {
748   MachineBasicBlock &MBB = *MI.getParent();
749   MachineFunction &MF = *MBB.getParent();
750   MachineFrameInfo &MFI = *MF.getFrameInfo();
751 
752   MFI.getObjectOffset(FrameRegIdx);
753   llvm_unreachable("Unimplemented rewriteFrameIndex");
754 }
755 
756 void llvm::emitRegUpdate(MachineBasicBlock &MBB,
757                          MachineBasicBlock::iterator MBBI,
758                          DebugLoc dl, const TargetInstrInfo &TII,
759                          unsigned DstReg, unsigned SrcReg, unsigned ScratchReg,
760                          int64_t NumBytes, MachineInstr::MIFlag MIFlags) {
761   if (NumBytes == 0 && DstReg == SrcReg)
762     return;
763   else if (abs64(NumBytes) & ~0xffffff) {
764     // Generically, we have to materialize the offset into a temporary register
765     // and subtract it. There are a couple of ways this could be done, for now
766     // we'll use a movz/movk or movn/movk sequence.
767     uint64_t Bits = static_cast<uint64_t>(abs64(NumBytes));
768     BuildMI(MBB, MBBI, dl, TII.get(AArch64::MOVZxii), ScratchReg)
769       .addImm(0xffff & Bits).addImm(0)
770       .setMIFlags(MIFlags);
771 
772     Bits >>= 16;
773     if (Bits & 0xffff) {
774       BuildMI(MBB, MBBI, dl, TII.get(AArch64::MOVKxii), ScratchReg)
775         .addReg(ScratchReg)
776         .addImm(0xffff & Bits).addImm(1)
777         .setMIFlags(MIFlags);
778     }
779 
780     Bits >>= 16;
781     if (Bits & 0xffff) {
782       BuildMI(MBB, MBBI, dl, TII.get(AArch64::MOVKxii), ScratchReg)
783         .addReg(ScratchReg)
784         .addImm(0xffff & Bits).addImm(2)
785         .setMIFlags(MIFlags);
786     }
787 
788     Bits >>= 16;
789     if (Bits & 0xffff) {
790       BuildMI(MBB, MBBI, dl, TII.get(AArch64::MOVKxii), ScratchReg)
791         .addReg(ScratchReg)
792         .addImm(0xffff & Bits).addImm(3)
793         .setMIFlags(MIFlags);
794     }
795 
796     // ADD DST, SRC, xTMP (, lsl #0)
797     unsigned AddOp = NumBytes > 0 ? AArch64::ADDxxx_uxtx : AArch64::SUBxxx_uxtx;
798     BuildMI(MBB, MBBI, dl, TII.get(AddOp), DstReg)
799       .addReg(SrcReg, RegState::Kill)
800       .addReg(ScratchReg, RegState::Kill)
801       .addImm(0)
802       .setMIFlag(MIFlags);
803     return;
804   }
805 
806   // Now we know that the adjustment can be done in at most two add/sub
807   // (immediate) instructions, which is always more efficient than a
808   // literal-pool load, or even a hypothetical movz/movk/add sequence
809 
810   // Decide whether we're doing addition or subtraction
811   unsigned LowOp, HighOp;
812   if (NumBytes >= 0) {
813     LowOp = AArch64::ADDxxi_lsl0_s;
814     HighOp = AArch64::ADDxxi_lsl12_s;
815   } else {
816     LowOp = AArch64::SUBxxi_lsl0_s;
817     HighOp = AArch64::SUBxxi_lsl12_s;
818     NumBytes = abs64(NumBytes);
819   }
820 
821   // If we're here, at the very least a move needs to be produced, which just
822   // happens to be materializable by an ADD.
823   if ((NumBytes & 0xfff) || NumBytes == 0) {
824     BuildMI(MBB, MBBI, dl, TII.get(LowOp), DstReg)
825       .addReg(SrcReg, RegState::Kill)
826       .addImm(NumBytes & 0xfff)
827       .setMIFlag(MIFlags);
828 
829     // Next update should use the register we've just defined.
830     SrcReg = DstReg;
831   }
832 
833   if (NumBytes & 0xfff000) {
834     BuildMI(MBB, MBBI, dl, TII.get(HighOp), DstReg)
835       .addReg(SrcReg, RegState::Kill)
836       .addImm(NumBytes >> 12)
837       .setMIFlag(MIFlags);
838   }
839 }
840 
841 void llvm::emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
842                         DebugLoc dl, const TargetInstrInfo &TII,
843                         unsigned ScratchReg, int64_t NumBytes,
844                         MachineInstr::MIFlag MIFlags) {
845   emitRegUpdate(MBB, MI, dl, TII, AArch64::XSP, AArch64::XSP, AArch64::X16,
846                 NumBytes, MIFlags);
847 }
848 
849 
850 namespace {
851   struct LDTLSCleanup : public MachineFunctionPass {
852     static char ID;
853     LDTLSCleanup() : MachineFunctionPass(ID) {}
854 
855     virtual bool runOnMachineFunction(MachineFunction &MF) {
856       AArch64MachineFunctionInfo* MFI
857         = MF.getInfo<AArch64MachineFunctionInfo>();
858       if (MFI->getNumLocalDynamicTLSAccesses() < 2) {
859         // No point folding accesses if there isn't at least two.
860         return false;
861       }
862 
863       MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>();
864       return VisitNode(DT->getRootNode(), 0);
865     }
866 
867     // Visit the dominator subtree rooted at Node in pre-order.
868     // If TLSBaseAddrReg is non-null, then use that to replace any
869     // TLS_base_addr instructions. Otherwise, create the register
870     // when the first such instruction is seen, and then use it
871     // as we encounter more instructions.
872     bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) {
873       MachineBasicBlock *BB = Node->getBlock();
874       bool Changed = false;
875 
876       // Traverse the current block.
877       for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;
878            ++I) {
879         switch (I->getOpcode()) {
880         case AArch64::TLSDESC_BLRx:
881           // Make sure it's a local dynamic access.
882           if (!I->getOperand(1).isSymbol() ||
883               strcmp(I->getOperand(1).getSymbolName(), "_TLS_MODULE_BASE_"))
884             break;
885 
886           if (TLSBaseAddrReg)
887             I = ReplaceTLSBaseAddrCall(I, TLSBaseAddrReg);
888           else
889             I = SetRegister(I, &TLSBaseAddrReg);
890           Changed = true;
891           break;
892         default:
893           break;
894         }
895       }
896 
897       // Visit the children of this block in the dominator tree.
898       for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end();
899            I != E; ++I) {
900         Changed |= VisitNode(*I, TLSBaseAddrReg);
901       }
902 
903       return Changed;
904     }
905 
906     // Replace the TLS_base_addr instruction I with a copy from
907     // TLSBaseAddrReg, returning the new instruction.
908     MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr *I,
909                                          unsigned TLSBaseAddrReg) {
910       MachineFunction *MF = I->getParent()->getParent();
911       const AArch64TargetMachine *TM =
912           static_cast<const AArch64TargetMachine *>(&MF->getTarget());
913       const AArch64InstrInfo *TII = TM->getInstrInfo();
914 
915       // Insert a Copy from TLSBaseAddrReg to x0, which is where the rest of the
916       // code sequence assumes the address will be.
917       MachineInstr *Copy = BuildMI(*I->getParent(), I, I->getDebugLoc(),
918                                    TII->get(TargetOpcode::COPY),
919                                    AArch64::X0)
920         .addReg(TLSBaseAddrReg);
921 
922       // Erase the TLS_base_addr instruction.
923       I->eraseFromParent();
924 
925       return Copy;
926     }
927 
928     // Create a virtal register in *TLSBaseAddrReg, and populate it by
929     // inserting a copy instruction after I. Returns the new instruction.
930     MachineInstr *SetRegister(MachineInstr *I, unsigned *TLSBaseAddrReg) {
931       MachineFunction *MF = I->getParent()->getParent();
932       const AArch64TargetMachine *TM =
933           static_cast<const AArch64TargetMachine *>(&MF->getTarget());
934       const AArch64InstrInfo *TII = TM->getInstrInfo();
935 
936       // Create a virtual register for the TLS base address.
937       MachineRegisterInfo &RegInfo = MF->getRegInfo();
938       *TLSBaseAddrReg = RegInfo.createVirtualRegister(&AArch64::GPR64RegClass);
939 
940       // Insert a copy from X0 to TLSBaseAddrReg for later.
941       MachineInstr *Next = I->getNextNode();
942       MachineInstr *Copy = BuildMI(*I->getParent(), Next, I->getDebugLoc(),
943                                    TII->get(TargetOpcode::COPY),
944                                    *TLSBaseAddrReg)
945         .addReg(AArch64::X0);
946 
947       return Copy;
948     }
949 
950     virtual const char *getPassName() const {
951       return "Local Dynamic TLS Access Clean-up";
952     }
953 
954     virtual void getAnalysisUsage(AnalysisUsage &AU) const {
955       AU.setPreservesCFG();
956       AU.addRequired<MachineDominatorTree>();
957       MachineFunctionPass::getAnalysisUsage(AU);
958     }
959   };
960 }
961 
962 char LDTLSCleanup::ID = 0;
963 FunctionPass*
964 llvm::createAArch64CleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); }
965