1 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64.h"
14 #include "AArch64CallLowering.h"
15 #include "AArch64InstructionSelector.h"
16 #include "AArch64LegalizerInfo.h"
17 #include "AArch64RegisterBankInfo.h"
18 #include "AArch64TargetMachine.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "AArch64TargetTransformInfo.h"
21 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
23 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
24 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
25 #include "llvm/CodeGen/MachineScheduler.h"
26 #include "llvm/CodeGen/Passes.h"
27 #include "llvm/CodeGen/RegAllocRegistry.h"
28 #include "llvm/CodeGen/TargetPassConfig.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/InitializePasses.h"
32 #include "llvm/Support/CommandLine.h"
33 #include "llvm/Support/TargetRegistry.h"
34 #include "llvm/Target/TargetOptions.h"
35 #include "llvm/Transforms/Scalar.h"
36 using namespace llvm;
37 
38 static cl::opt<bool> EnableCCMP("aarch64-enable-ccmp",
39                                 cl::desc("Enable the CCMP formation pass"),
40                                 cl::init(true), cl::Hidden);
41 
42 static cl::opt<bool> EnableMCR("aarch64-enable-mcr",
43                                cl::desc("Enable the machine combiner pass"),
44                                cl::init(true), cl::Hidden);
45 
46 static cl::opt<bool> EnableStPairSuppress("aarch64-enable-stp-suppress",
47                                           cl::desc("Suppress STP for AArch64"),
48                                           cl::init(true), cl::Hidden);
49 
50 static cl::opt<bool> EnableAdvSIMDScalar(
51     "aarch64-enable-simd-scalar",
52     cl::desc("Enable use of AdvSIMD scalar integer instructions"),
53     cl::init(false), cl::Hidden);
54 
55 static cl::opt<bool>
56     EnablePromoteConstant("aarch64-enable-promote-const",
57                           cl::desc("Enable the promote constant pass"),
58                           cl::init(true), cl::Hidden);
59 
60 static cl::opt<bool> EnableCollectLOH(
61     "aarch64-enable-collect-loh",
62     cl::desc("Enable the pass that emits the linker optimization hints (LOH)"),
63     cl::init(true), cl::Hidden);
64 
65 static cl::opt<bool>
66     EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden,
67                                   cl::desc("Enable the pass that removes dead"
68                                            " definitons and replaces stores to"
69                                            " them with stores to the zero"
70                                            " register"),
71                                   cl::init(true));
72 
73 static cl::opt<bool> EnableRedundantCopyElimination(
74     "aarch64-enable-copyelim",
75     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
76     cl::Hidden);
77 
78 static cl::opt<bool> EnableLoadStoreOpt("aarch64-enable-ldst-opt",
79                                         cl::desc("Enable the load/store pair"
80                                                  " optimization pass"),
81                                         cl::init(true), cl::Hidden);
82 
83 static cl::opt<bool> EnableAtomicTidy(
84     "aarch64-enable-atomic-cfg-tidy", cl::Hidden,
85     cl::desc("Run SimplifyCFG after expanding atomic operations"
86              " to make use of cmpxchg flow-based information"),
87     cl::init(true));
88 
89 static cl::opt<bool>
90 EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden,
91                         cl::desc("Run early if-conversion"),
92                         cl::init(true));
93 
94 static cl::opt<bool>
95     EnableCondOpt("aarch64-enable-condopt",
96                   cl::desc("Enable the condition optimizer pass"),
97                   cl::init(true), cl::Hidden);
98 
99 static cl::opt<bool>
100 EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden,
101                 cl::desc("Work around Cortex-A53 erratum 835769"),
102                 cl::init(false));
103 
104 static cl::opt<bool>
105     EnableAddressTypePromotion("aarch64-enable-type-promotion", cl::Hidden,
106                                cl::desc("Enable the type promotion pass"),
107                                cl::init(true));
108 
109 static cl::opt<bool>
110     EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden,
111                  cl::desc("Enable optimizations on complex GEPs"),
112                  cl::init(false));
113 
114 static cl::opt<bool>
115     BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true),
116                      cl::desc("Relax out of range conditional branches"));
117 
118 // FIXME: Unify control over GlobalMerge.
119 static cl::opt<cl::boolOrDefault>
120     EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden,
121                       cl::desc("Enable the global merge pass"));
122 
123 static cl::opt<bool>
124     EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden,
125                            cl::desc("Enable the loop data prefetch pass"),
126                            cl::init(true));
127 
128 extern "C" void LLVMInitializeAArch64Target() {
129   // Register the target.
130   RegisterTargetMachine<AArch64leTargetMachine> X(getTheAArch64leTarget());
131   RegisterTargetMachine<AArch64beTargetMachine> Y(getTheAArch64beTarget());
132   RegisterTargetMachine<AArch64leTargetMachine> Z(getTheARM64Target());
133   auto PR = PassRegistry::getPassRegistry();
134   initializeGlobalISel(*PR);
135   initializeAArch64A53Fix835769Pass(*PR);
136   initializeAArch64A57FPLoadBalancingPass(*PR);
137   initializeAArch64AddressTypePromotionPass(*PR);
138   initializeAArch64AdvSIMDScalarPass(*PR);
139   initializeAArch64CollectLOHPass(*PR);
140   initializeAArch64ConditionalComparesPass(*PR);
141   initializeAArch64ConditionOptimizerPass(*PR);
142   initializeAArch64DeadRegisterDefinitionsPass(*PR);
143   initializeAArch64ExpandPseudoPass(*PR);
144   initializeAArch64LoadStoreOptPass(*PR);
145   initializeAArch64VectorByElementOptPass(*PR);
146   initializeAArch64PromoteConstantPass(*PR);
147   initializeAArch64RedundantCopyEliminationPass(*PR);
148   initializeAArch64StorePairSuppressPass(*PR);
149   initializeLDTLSCleanupPass(*PR);
150 }
151 
152 //===----------------------------------------------------------------------===//
153 // AArch64 Lowering public interface.
154 //===----------------------------------------------------------------------===//
155 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
156   if (TT.isOSBinFormatMachO())
157     return make_unique<AArch64_MachoTargetObjectFile>();
158 
159   return make_unique<AArch64_ELFTargetObjectFile>();
160 }
161 
162 // Helper function to build a DataLayout string
163 static std::string computeDataLayout(const Triple &TT,
164                                      const MCTargetOptions &Options,
165                                      bool LittleEndian) {
166   if (Options.getABIName() == "ilp32")
167     return "e-m:e-p:32:32-i8:8-i16:16-i64:64-S128";
168   if (TT.isOSBinFormatMachO())
169     return "e-m:o-i64:64-i128:128-n32:64-S128";
170   if (LittleEndian)
171     return "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128";
172   return "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128";
173 }
174 
175 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
176                                            Optional<Reloc::Model> RM) {
177   // AArch64 Darwin is always PIC.
178   if (TT.isOSDarwin())
179     return Reloc::PIC_;
180   // On ELF platforms the default static relocation model has a smart enough
181   // linker to cope with referencing external symbols defined in a shared
182   // library. Hence DynamicNoPIC doesn't need to be promoted to PIC.
183   if (!RM.hasValue() || *RM == Reloc::DynamicNoPIC)
184     return Reloc::Static;
185   return *RM;
186 }
187 
188 /// Create an AArch64 architecture model.
189 ///
190 AArch64TargetMachine::AArch64TargetMachine(
191     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
192     const TargetOptions &Options, Optional<Reloc::Model> RM,
193     CodeModel::Model CM, CodeGenOpt::Level OL, bool LittleEndian)
194     // This nested ternary is horrible, but DL needs to be properly
195     // initialized before TLInfo is constructed.
196     : LLVMTargetMachine(T, computeDataLayout(TT, Options.MCOptions,
197                                              LittleEndian),
198                         TT, CPU, FS, Options,
199 			getEffectiveRelocModel(TT, RM), CM, OL),
200       TLOF(createTLOF(getTargetTriple())),
201       isLittle(LittleEndian) {
202   initAsmInfo();
203 }
204 
205 AArch64TargetMachine::~AArch64TargetMachine() {}
206 
207 #ifdef LLVM_BUILD_GLOBAL_ISEL
208 namespace {
209 struct AArch64GISelActualAccessor : public GISelAccessor {
210   std::unique_ptr<CallLowering> CallLoweringInfo;
211   std::unique_ptr<InstructionSelector> InstSelector;
212   std::unique_ptr<LegalizerInfo> Legalizer;
213   std::unique_ptr<RegisterBankInfo> RegBankInfo;
214   const CallLowering *getCallLowering() const override {
215     return CallLoweringInfo.get();
216   }
217   const InstructionSelector *getInstructionSelector() const override {
218     return InstSelector.get();
219   }
220   const class LegalizerInfo *getLegalizerInfo() const override {
221     return Legalizer.get();
222   }
223   const RegisterBankInfo *getRegBankInfo() const override {
224     return RegBankInfo.get();
225   }
226 };
227 } // End anonymous namespace.
228 #endif
229 
230 const AArch64Subtarget *
231 AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
232   Attribute CPUAttr = F.getFnAttribute("target-cpu");
233   Attribute FSAttr = F.getFnAttribute("target-features");
234 
235   std::string CPU = !CPUAttr.hasAttribute(Attribute::None)
236                         ? CPUAttr.getValueAsString().str()
237                         : TargetCPU;
238   std::string FS = !FSAttr.hasAttribute(Attribute::None)
239                        ? FSAttr.getValueAsString().str()
240                        : TargetFS;
241 
242   auto &I = SubtargetMap[CPU + FS];
243   if (!I) {
244     // This needs to be done before we create a new subtarget since any
245     // creation will depend on the TM and the code generation flags on the
246     // function that reside in TargetOptions.
247     resetTargetOptions(F);
248     I = llvm::make_unique<AArch64Subtarget>(TargetTriple, CPU, FS, *this,
249                                             isLittle);
250 #ifndef LLVM_BUILD_GLOBAL_ISEL
251     GISelAccessor *GISel = new GISelAccessor();
252 #else
253     AArch64GISelActualAccessor *GISel =
254         new AArch64GISelActualAccessor();
255     GISel->CallLoweringInfo.reset(
256         new AArch64CallLowering(*I->getTargetLowering()));
257     GISel->Legalizer.reset(new AArch64LegalizerInfo());
258 
259     auto *RBI = new AArch64RegisterBankInfo(*I->getRegisterInfo());
260 
261     // FIXME: At this point, we can't rely on Subtarget having RBI.
262     // It's awkward to mix passing RBI and the Subtarget; should we pass
263     // TII/TRI as well?
264     GISel->InstSelector.reset(new AArch64InstructionSelector(*this, *I, *RBI));
265 
266     GISel->RegBankInfo.reset(RBI);
267 #endif
268     I->setGISelAccessor(*GISel);
269   }
270   return I.get();
271 }
272 
273 void AArch64leTargetMachine::anchor() { }
274 
275 AArch64leTargetMachine::AArch64leTargetMachine(
276     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
277     const TargetOptions &Options, Optional<Reloc::Model> RM,
278     CodeModel::Model CM, CodeGenOpt::Level OL)
279     : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
280 
281 void AArch64beTargetMachine::anchor() { }
282 
283 AArch64beTargetMachine::AArch64beTargetMachine(
284     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
285     const TargetOptions &Options, Optional<Reloc::Model> RM,
286     CodeModel::Model CM, CodeGenOpt::Level OL)
287     : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
288 
289 namespace {
290 /// AArch64 Code Generator Pass Configuration Options.
291 class AArch64PassConfig : public TargetPassConfig {
292 public:
293   AArch64PassConfig(AArch64TargetMachine *TM, PassManagerBase &PM)
294       : TargetPassConfig(TM, PM) {
295     if (TM->getOptLevel() != CodeGenOpt::None)
296       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
297   }
298 
299   AArch64TargetMachine &getAArch64TargetMachine() const {
300     return getTM<AArch64TargetMachine>();
301   }
302 
303   ScheduleDAGInstrs *
304   createMachineScheduler(MachineSchedContext *C) const override {
305     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
306     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
307     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
308     DAG->addMutation(createMacroFusionDAGMutation(DAG->TII));
309     return DAG;
310   }
311 
312   void addIRPasses()  override;
313   bool addPreISel() override;
314   bool addInstSelector() override;
315 #ifdef LLVM_BUILD_GLOBAL_ISEL
316   bool addIRTranslator() override;
317   bool addLegalizeMachineIR() override;
318   bool addRegBankSelect() override;
319   bool addGlobalInstructionSelect() override;
320 #endif
321   bool addILPOpts() override;
322   void addPreRegAlloc() override;
323   void addPostRegAlloc() override;
324   void addPreSched2() override;
325   void addPreEmitPass() override;
326 };
327 } // namespace
328 
329 TargetIRAnalysis AArch64TargetMachine::getTargetIRAnalysis() {
330   return TargetIRAnalysis([this](const Function &F) {
331     return TargetTransformInfo(AArch64TTIImpl(this, F));
332   });
333 }
334 
335 TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) {
336   return new AArch64PassConfig(this, PM);
337 }
338 
339 void AArch64PassConfig::addIRPasses() {
340   // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg
341   // ourselves.
342   addPass(createAtomicExpandPass(TM));
343 
344   // Cmpxchg instructions are often used with a subsequent comparison to
345   // determine whether it succeeded. We can exploit existing control-flow in
346   // ldrex/strex loops to simplify this, but it needs tidying up.
347   if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
348     addPass(createCFGSimplificationPass());
349 
350   // Run LoopDataPrefetch
351   //
352   // Run this before LSR to remove the multiplies involved in computing the
353   // pointer values N iterations ahead.
354   if (TM->getOptLevel() != CodeGenOpt::None && EnableLoopDataPrefetch)
355     addPass(createLoopDataPrefetchPass());
356 
357   TargetPassConfig::addIRPasses();
358 
359   // Match interleaved memory accesses to ldN/stN intrinsics.
360   if (TM->getOptLevel() != CodeGenOpt::None)
361     addPass(createInterleavedAccessPass(TM));
362 
363   if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) {
364     // Call SeparateConstOffsetFromGEP pass to extract constants within indices
365     // and lower a GEP with multiple indices to either arithmetic operations or
366     // multiple GEPs with single index.
367     addPass(createSeparateConstOffsetFromGEPPass(TM, true));
368     // Call EarlyCSE pass to find and remove subexpressions in the lowered
369     // result.
370     addPass(createEarlyCSEPass());
371     // Do loop invariant code motion in case part of the lowered result is
372     // invariant.
373     addPass(createLICMPass());
374   }
375 }
376 
377 // Pass Pipeline Configuration
378 bool AArch64PassConfig::addPreISel() {
379   // Run promote constant before global merge, so that the promoted constants
380   // get a chance to be merged
381   if (TM->getOptLevel() != CodeGenOpt::None && EnablePromoteConstant)
382     addPass(createAArch64PromoteConstantPass());
383   // FIXME: On AArch64, this depends on the type.
384   // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes().
385   // and the offset has to be a multiple of the related size in bytes.
386   if ((TM->getOptLevel() != CodeGenOpt::None &&
387        EnableGlobalMerge == cl::BOU_UNSET) ||
388       EnableGlobalMerge == cl::BOU_TRUE) {
389     bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) &&
390                                (EnableGlobalMerge == cl::BOU_UNSET);
391     addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize));
392   }
393 
394   if (TM->getOptLevel() != CodeGenOpt::None && EnableAddressTypePromotion)
395     addPass(createAArch64AddressTypePromotionPass());
396 
397   return false;
398 }
399 
400 bool AArch64PassConfig::addInstSelector() {
401   addPass(createAArch64ISelDag(getAArch64TargetMachine(), getOptLevel()));
402 
403   // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many
404   // references to _TLS_MODULE_BASE_ as possible.
405   if (TM->getTargetTriple().isOSBinFormatELF() &&
406       getOptLevel() != CodeGenOpt::None)
407     addPass(createAArch64CleanupLocalDynamicTLSPass());
408 
409   return false;
410 }
411 
412 #ifdef LLVM_BUILD_GLOBAL_ISEL
413 bool AArch64PassConfig::addIRTranslator() {
414   addPass(new IRTranslator());
415   return false;
416 }
417 bool AArch64PassConfig::addLegalizeMachineIR() {
418   addPass(new Legalizer());
419   return false;
420 }
421 bool AArch64PassConfig::addRegBankSelect() {
422   addPass(new RegBankSelect());
423   return false;
424 }
425 bool AArch64PassConfig::addGlobalInstructionSelect() {
426   addPass(new InstructionSelect());
427   return false;
428 }
429 #endif
430 
431 bool AArch64PassConfig::addILPOpts() {
432   if (EnableCondOpt)
433     addPass(createAArch64ConditionOptimizerPass());
434   if (EnableCCMP)
435     addPass(createAArch64ConditionalCompares());
436   if (EnableMCR)
437     addPass(&MachineCombinerID);
438   if (EnableEarlyIfConversion)
439     addPass(&EarlyIfConverterID);
440   if (EnableStPairSuppress)
441     addPass(createAArch64StorePairSuppressPass());
442   addPass(createAArch64VectorByElementOptPass());
443   return true;
444 }
445 
446 void AArch64PassConfig::addPreRegAlloc() {
447   // Change dead register definitions to refer to the zero register.
448   if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination)
449     addPass(createAArch64DeadRegisterDefinitions());
450 
451   // Use AdvSIMD scalar instructions whenever profitable.
452   if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) {
453     addPass(createAArch64AdvSIMDScalar());
454     // The AdvSIMD pass may produce copies that can be rewritten to
455     // be register coaleascer friendly.
456     addPass(&PeepholeOptimizerID);
457   }
458 }
459 
460 void AArch64PassConfig::addPostRegAlloc() {
461   // Remove redundant copy instructions.
462   if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination)
463     addPass(createAArch64RedundantCopyEliminationPass());
464 
465   if (TM->getOptLevel() != CodeGenOpt::None && usingDefaultRegAlloc())
466     // Improve performance for some FP/SIMD code for A57.
467     addPass(createAArch64A57FPLoadBalancing());
468 }
469 
470 void AArch64PassConfig::addPreSched2() {
471   // Expand some pseudo instructions to allow proper scheduling.
472   addPass(createAArch64ExpandPseudoPass());
473   // Use load/store pair instructions when possible.
474   if (TM->getOptLevel() != CodeGenOpt::None && EnableLoadStoreOpt)
475     addPass(createAArch64LoadStoreOptimizationPass());
476 }
477 
478 void AArch64PassConfig::addPreEmitPass() {
479   if (EnableA53Fix835769)
480     addPass(createAArch64A53Fix835769());
481   // Relax conditional branch instructions if they're otherwise out of
482   // range of their destination.
483   if (BranchRelaxation)
484     addPass(&BranchRelaxationPassID);
485 
486   if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH &&
487       TM->getTargetTriple().isOSBinFormatMachO())
488     addPass(createAArch64CollectLOHPass());
489 }
490