1 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64.h"
14 #include "AArch64CallLowering.h"
15 #include "AArch64InstructionSelector.h"
16 #include "AArch64LegalizerInfo.h"
17 #ifdef LLVM_BUILD_GLOBAL_ISEL
18 #include "AArch64RegisterBankInfo.h"
19 #endif
20 #include "AArch64Subtarget.h"
21 #include "AArch64TargetMachine.h"
22 #include "AArch64TargetObjectFile.h"
23 #include "AArch64TargetTransformInfo.h"
24 #include "MCTargetDesc/AArch64MCTargetDesc.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/Triple.h"
27 #include "llvm/Analysis/TargetTransformInfo.h"
28 #include "llvm/CodeGen/GlobalISel/GISelAccessor.h"
29 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
30 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
31 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
32 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
33 #include "llvm/CodeGen/MachineScheduler.h"
34 #include "llvm/CodeGen/Passes.h"
35 #include "llvm/CodeGen/TargetPassConfig.h"
36 #include "llvm/IR/Attributes.h"
37 #include "llvm/IR/Function.h"
38 #include "llvm/MC/MCTargetOptions.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/TargetRegistry.h"
43 #include "llvm/Target/TargetLoweringObjectFile.h"
44 #include "llvm/Target/TargetOptions.h"
45 #include "llvm/Transforms/Scalar.h"
46 #include <memory>
47 #include <string>
48 
49 using namespace llvm;
50 
51 static cl::opt<bool> EnableCCMP("aarch64-enable-ccmp",
52                                 cl::desc("Enable the CCMP formation pass"),
53                                 cl::init(true), cl::Hidden);
54 
55 static cl::opt<bool> EnableMCR("aarch64-enable-mcr",
56                                cl::desc("Enable the machine combiner pass"),
57                                cl::init(true), cl::Hidden);
58 
59 static cl::opt<bool> EnableStPairSuppress("aarch64-enable-stp-suppress",
60                                           cl::desc("Suppress STP for AArch64"),
61                                           cl::init(true), cl::Hidden);
62 
63 static cl::opt<bool> EnableAdvSIMDScalar(
64     "aarch64-enable-simd-scalar",
65     cl::desc("Enable use of AdvSIMD scalar integer instructions"),
66     cl::init(false), cl::Hidden);
67 
68 static cl::opt<bool>
69     EnablePromoteConstant("aarch64-enable-promote-const",
70                           cl::desc("Enable the promote constant pass"),
71                           cl::init(true), cl::Hidden);
72 
73 static cl::opt<bool> EnableCollectLOH(
74     "aarch64-enable-collect-loh",
75     cl::desc("Enable the pass that emits the linker optimization hints (LOH)"),
76     cl::init(true), cl::Hidden);
77 
78 static cl::opt<bool>
79     EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden,
80                                   cl::desc("Enable the pass that removes dead"
81                                            " definitons and replaces stores to"
82                                            " them with stores to the zero"
83                                            " register"),
84                                   cl::init(true));
85 
86 static cl::opt<bool> EnableRedundantCopyElimination(
87     "aarch64-enable-copyelim",
88     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
89     cl::Hidden);
90 
91 static cl::opt<bool> EnableLoadStoreOpt("aarch64-enable-ldst-opt",
92                                         cl::desc("Enable the load/store pair"
93                                                  " optimization pass"),
94                                         cl::init(true), cl::Hidden);
95 
96 static cl::opt<bool> EnableAtomicTidy(
97     "aarch64-enable-atomic-cfg-tidy", cl::Hidden,
98     cl::desc("Run SimplifyCFG after expanding atomic operations"
99              " to make use of cmpxchg flow-based information"),
100     cl::init(true));
101 
102 static cl::opt<bool>
103 EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden,
104                         cl::desc("Run early if-conversion"),
105                         cl::init(true));
106 
107 static cl::opt<bool>
108     EnableCondOpt("aarch64-enable-condopt",
109                   cl::desc("Enable the condition optimizer pass"),
110                   cl::init(true), cl::Hidden);
111 
112 static cl::opt<bool>
113 EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden,
114                 cl::desc("Work around Cortex-A53 erratum 835769"),
115                 cl::init(false));
116 
117 static cl::opt<bool>
118     EnableAddressTypePromotion("aarch64-enable-type-promotion", cl::Hidden,
119                                cl::desc("Enable the type promotion pass"),
120                                cl::init(true));
121 
122 static cl::opt<bool>
123     EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden,
124                  cl::desc("Enable optimizations on complex GEPs"),
125                  cl::init(false));
126 
127 static cl::opt<bool>
128     BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true),
129                      cl::desc("Relax out of range conditional branches"));
130 
131 // FIXME: Unify control over GlobalMerge.
132 static cl::opt<cl::boolOrDefault>
133     EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden,
134                       cl::desc("Enable the global merge pass"));
135 
136 static cl::opt<bool>
137     EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden,
138                            cl::desc("Enable the loop data prefetch pass"),
139                            cl::init(true));
140 
141 extern "C" void LLVMInitializeAArch64Target() {
142   // Register the target.
143   RegisterTargetMachine<AArch64leTargetMachine> X(getTheAArch64leTarget());
144   RegisterTargetMachine<AArch64beTargetMachine> Y(getTheAArch64beTarget());
145   RegisterTargetMachine<AArch64leTargetMachine> Z(getTheARM64Target());
146   auto PR = PassRegistry::getPassRegistry();
147   initializeGlobalISel(*PR);
148   initializeAArch64A53Fix835769Pass(*PR);
149   initializeAArch64A57FPLoadBalancingPass(*PR);
150   initializeAArch64AddressTypePromotionPass(*PR);
151   initializeAArch64AdvSIMDScalarPass(*PR);
152   initializeAArch64CollectLOHPass(*PR);
153   initializeAArch64ConditionalComparesPass(*PR);
154   initializeAArch64ConditionOptimizerPass(*PR);
155   initializeAArch64DeadRegisterDefinitionsPass(*PR);
156   initializeAArch64ExpandPseudoPass(*PR);
157   initializeAArch64LoadStoreOptPass(*PR);
158   initializeAArch64VectorByElementOptPass(*PR);
159   initializeAArch64PromoteConstantPass(*PR);
160   initializeAArch64RedundantCopyEliminationPass(*PR);
161   initializeAArch64StorePairSuppressPass(*PR);
162   initializeLDTLSCleanupPass(*PR);
163 }
164 
165 //===----------------------------------------------------------------------===//
166 // AArch64 Lowering public interface.
167 //===----------------------------------------------------------------------===//
168 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
169   if (TT.isOSBinFormatMachO())
170     return llvm::make_unique<AArch64_MachoTargetObjectFile>();
171 
172   return llvm::make_unique<AArch64_ELFTargetObjectFile>();
173 }
174 
175 // Helper function to build a DataLayout string
176 static std::string computeDataLayout(const Triple &TT,
177                                      const MCTargetOptions &Options,
178                                      bool LittleEndian) {
179   if (Options.getABIName() == "ilp32")
180     return "e-m:e-p:32:32-i8:8-i16:16-i64:64-S128";
181   if (TT.isOSBinFormatMachO())
182     return "e-m:o-i64:64-i128:128-n32:64-S128";
183   if (LittleEndian)
184     return "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128";
185   return "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128";
186 }
187 
188 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
189                                            Optional<Reloc::Model> RM) {
190   // AArch64 Darwin is always PIC.
191   if (TT.isOSDarwin())
192     return Reloc::PIC_;
193   // On ELF platforms the default static relocation model has a smart enough
194   // linker to cope with referencing external symbols defined in a shared
195   // library. Hence DynamicNoPIC doesn't need to be promoted to PIC.
196   if (!RM.hasValue() || *RM == Reloc::DynamicNoPIC)
197     return Reloc::Static;
198   return *RM;
199 }
200 
201 /// Create an AArch64 architecture model.
202 ///
203 AArch64TargetMachine::AArch64TargetMachine(
204     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
205     const TargetOptions &Options, Optional<Reloc::Model> RM,
206     CodeModel::Model CM, CodeGenOpt::Level OL, bool LittleEndian)
207     // This nested ternary is horrible, but DL needs to be properly
208     // initialized before TLInfo is constructed.
209     : LLVMTargetMachine(T, computeDataLayout(TT, Options.MCOptions,
210                                              LittleEndian),
211                         TT, CPU, FS, Options,
212 			getEffectiveRelocModel(TT, RM), CM, OL),
213       TLOF(createTLOF(getTargetTriple())),
214       isLittle(LittleEndian) {
215   initAsmInfo();
216 }
217 
218 AArch64TargetMachine::~AArch64TargetMachine() = default;
219 
220 #ifdef LLVM_BUILD_GLOBAL_ISEL
221 namespace {
222 
223 struct AArch64GISelActualAccessor : public GISelAccessor {
224   std::unique_ptr<CallLowering> CallLoweringInfo;
225   std::unique_ptr<InstructionSelector> InstSelector;
226   std::unique_ptr<LegalizerInfo> Legalizer;
227   std::unique_ptr<RegisterBankInfo> RegBankInfo;
228 
229   const CallLowering *getCallLowering() const override {
230     return CallLoweringInfo.get();
231   }
232 
233   const InstructionSelector *getInstructionSelector() const override {
234     return InstSelector.get();
235   }
236 
237   const LegalizerInfo *getLegalizerInfo() const override {
238     return Legalizer.get();
239   }
240 
241   const RegisterBankInfo *getRegBankInfo() const override {
242     return RegBankInfo.get();
243   }
244 };
245 
246 } // end anonymous namespace
247 #endif
248 
249 const AArch64Subtarget *
250 AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
251   Attribute CPUAttr = F.getFnAttribute("target-cpu");
252   Attribute FSAttr = F.getFnAttribute("target-features");
253 
254   std::string CPU = !CPUAttr.hasAttribute(Attribute::None)
255                         ? CPUAttr.getValueAsString().str()
256                         : TargetCPU;
257   std::string FS = !FSAttr.hasAttribute(Attribute::None)
258                        ? FSAttr.getValueAsString().str()
259                        : TargetFS;
260 
261   auto &I = SubtargetMap[CPU + FS];
262   if (!I) {
263     // This needs to be done before we create a new subtarget since any
264     // creation will depend on the TM and the code generation flags on the
265     // function that reside in TargetOptions.
266     resetTargetOptions(F);
267     I = llvm::make_unique<AArch64Subtarget>(TargetTriple, CPU, FS, *this,
268                                             isLittle);
269 #ifndef LLVM_BUILD_GLOBAL_ISEL
270     GISelAccessor *GISel = new GISelAccessor();
271 #else
272     AArch64GISelActualAccessor *GISel =
273         new AArch64GISelActualAccessor();
274     GISel->CallLoweringInfo.reset(
275         new AArch64CallLowering(*I->getTargetLowering()));
276     GISel->Legalizer.reset(new AArch64LegalizerInfo());
277 
278     auto *RBI = new AArch64RegisterBankInfo(*I->getRegisterInfo());
279 
280     // FIXME: At this point, we can't rely on Subtarget having RBI.
281     // It's awkward to mix passing RBI and the Subtarget; should we pass
282     // TII/TRI as well?
283     GISel->InstSelector.reset(new AArch64InstructionSelector(*this, *I, *RBI));
284 
285     GISel->RegBankInfo.reset(RBI);
286 #endif
287     I->setGISelAccessor(*GISel);
288   }
289   return I.get();
290 }
291 
292 void AArch64leTargetMachine::anchor() { }
293 
294 AArch64leTargetMachine::AArch64leTargetMachine(
295     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
296     const TargetOptions &Options, Optional<Reloc::Model> RM,
297     CodeModel::Model CM, CodeGenOpt::Level OL)
298     : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
299 
300 void AArch64beTargetMachine::anchor() { }
301 
302 AArch64beTargetMachine::AArch64beTargetMachine(
303     const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
304     const TargetOptions &Options, Optional<Reloc::Model> RM,
305     CodeModel::Model CM, CodeGenOpt::Level OL)
306     : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
307 
308 namespace {
309 
310 /// AArch64 Code Generator Pass Configuration Options.
311 class AArch64PassConfig : public TargetPassConfig {
312 public:
313   AArch64PassConfig(AArch64TargetMachine *TM, PassManagerBase &PM)
314       : TargetPassConfig(TM, PM) {
315     if (TM->getOptLevel() != CodeGenOpt::None)
316       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
317   }
318 
319   AArch64TargetMachine &getAArch64TargetMachine() const {
320     return getTM<AArch64TargetMachine>();
321   }
322 
323   ScheduleDAGInstrs *
324   createMachineScheduler(MachineSchedContext *C) const override {
325     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
326     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
327     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
328     DAG->addMutation(createMacroFusionDAGMutation(DAG->TII));
329     return DAG;
330   }
331 
332   void addIRPasses()  override;
333   bool addPreISel() override;
334   bool addInstSelector() override;
335 #ifdef LLVM_BUILD_GLOBAL_ISEL
336   bool addIRTranslator() override;
337   bool addLegalizeMachineIR() override;
338   bool addRegBankSelect() override;
339   bool addGlobalInstructionSelect() override;
340 #endif
341   bool addILPOpts() override;
342   void addPreRegAlloc() override;
343   void addPostRegAlloc() override;
344   void addPreSched2() override;
345   void addPreEmitPass() override;
346 };
347 
348 } // end anonymous namespace
349 
350 TargetIRAnalysis AArch64TargetMachine::getTargetIRAnalysis() {
351   return TargetIRAnalysis([this](const Function &F) {
352     return TargetTransformInfo(AArch64TTIImpl(this, F));
353   });
354 }
355 
356 TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) {
357   return new AArch64PassConfig(this, PM);
358 }
359 
360 void AArch64PassConfig::addIRPasses() {
361   // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg
362   // ourselves.
363   addPass(createAtomicExpandPass(TM));
364 
365   // Cmpxchg instructions are often used with a subsequent comparison to
366   // determine whether it succeeded. We can exploit existing control-flow in
367   // ldrex/strex loops to simplify this, but it needs tidying up.
368   if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
369     addPass(createCFGSimplificationPass());
370 
371   // Run LoopDataPrefetch
372   //
373   // Run this before LSR to remove the multiplies involved in computing the
374   // pointer values N iterations ahead.
375   if (TM->getOptLevel() != CodeGenOpt::None && EnableLoopDataPrefetch)
376     addPass(createLoopDataPrefetchPass());
377 
378   TargetPassConfig::addIRPasses();
379 
380   // Match interleaved memory accesses to ldN/stN intrinsics.
381   if (TM->getOptLevel() != CodeGenOpt::None)
382     addPass(createInterleavedAccessPass(TM));
383 
384   if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) {
385     // Call SeparateConstOffsetFromGEP pass to extract constants within indices
386     // and lower a GEP with multiple indices to either arithmetic operations or
387     // multiple GEPs with single index.
388     addPass(createSeparateConstOffsetFromGEPPass(TM, true));
389     // Call EarlyCSE pass to find and remove subexpressions in the lowered
390     // result.
391     addPass(createEarlyCSEPass());
392     // Do loop invariant code motion in case part of the lowered result is
393     // invariant.
394     addPass(createLICMPass());
395   }
396 }
397 
398 // Pass Pipeline Configuration
399 bool AArch64PassConfig::addPreISel() {
400   // Run promote constant before global merge, so that the promoted constants
401   // get a chance to be merged
402   if (TM->getOptLevel() != CodeGenOpt::None && EnablePromoteConstant)
403     addPass(createAArch64PromoteConstantPass());
404   // FIXME: On AArch64, this depends on the type.
405   // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes().
406   // and the offset has to be a multiple of the related size in bytes.
407   if ((TM->getOptLevel() != CodeGenOpt::None &&
408        EnableGlobalMerge == cl::BOU_UNSET) ||
409       EnableGlobalMerge == cl::BOU_TRUE) {
410     bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) &&
411                                (EnableGlobalMerge == cl::BOU_UNSET);
412     addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize));
413   }
414 
415   if (TM->getOptLevel() != CodeGenOpt::None && EnableAddressTypePromotion)
416     addPass(createAArch64AddressTypePromotionPass());
417 
418   return false;
419 }
420 
421 bool AArch64PassConfig::addInstSelector() {
422   addPass(createAArch64ISelDag(getAArch64TargetMachine(), getOptLevel()));
423 
424   // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many
425   // references to _TLS_MODULE_BASE_ as possible.
426   if (TM->getTargetTriple().isOSBinFormatELF() &&
427       getOptLevel() != CodeGenOpt::None)
428     addPass(createAArch64CleanupLocalDynamicTLSPass());
429 
430   return false;
431 }
432 
433 #ifdef LLVM_BUILD_GLOBAL_ISEL
434 bool AArch64PassConfig::addIRTranslator() {
435   addPass(new IRTranslator());
436   return false;
437 }
438 
439 bool AArch64PassConfig::addLegalizeMachineIR() {
440   addPass(new Legalizer());
441   return false;
442 }
443 
444 bool AArch64PassConfig::addRegBankSelect() {
445   addPass(new RegBankSelect());
446   return false;
447 }
448 
449 bool AArch64PassConfig::addGlobalInstructionSelect() {
450   addPass(new InstructionSelect());
451   return false;
452 }
453 #endif
454 
455 bool AArch64PassConfig::addILPOpts() {
456   if (EnableCondOpt)
457     addPass(createAArch64ConditionOptimizerPass());
458   if (EnableCCMP)
459     addPass(createAArch64ConditionalCompares());
460   if (EnableMCR)
461     addPass(&MachineCombinerID);
462   if (EnableEarlyIfConversion)
463     addPass(&EarlyIfConverterID);
464   if (EnableStPairSuppress)
465     addPass(createAArch64StorePairSuppressPass());
466   addPass(createAArch64VectorByElementOptPass());
467   return true;
468 }
469 
470 void AArch64PassConfig::addPreRegAlloc() {
471   // Change dead register definitions to refer to the zero register.
472   if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination)
473     addPass(createAArch64DeadRegisterDefinitions());
474 
475   // Use AdvSIMD scalar instructions whenever profitable.
476   if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) {
477     addPass(createAArch64AdvSIMDScalar());
478     // The AdvSIMD pass may produce copies that can be rewritten to
479     // be register coaleascer friendly.
480     addPass(&PeepholeOptimizerID);
481   }
482 }
483 
484 void AArch64PassConfig::addPostRegAlloc() {
485   // Remove redundant copy instructions.
486   if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination)
487     addPass(createAArch64RedundantCopyEliminationPass());
488 
489   if (TM->getOptLevel() != CodeGenOpt::None && usingDefaultRegAlloc())
490     // Improve performance for some FP/SIMD code for A57.
491     addPass(createAArch64A57FPLoadBalancing());
492 }
493 
494 void AArch64PassConfig::addPreSched2() {
495   // Expand some pseudo instructions to allow proper scheduling.
496   addPass(createAArch64ExpandPseudoPass());
497   // Use load/store pair instructions when possible.
498   if (TM->getOptLevel() != CodeGenOpt::None && EnableLoadStoreOpt)
499     addPass(createAArch64LoadStoreOptimizationPass());
500 }
501 
502 void AArch64PassConfig::addPreEmitPass() {
503   if (EnableA53Fix835769)
504     addPass(createAArch64A53Fix835769());
505   // Relax conditional branch instructions if they're otherwise out of
506   // range of their destination.
507   if (BranchRelaxation)
508     addPass(&BranchRelaxationPassID);
509 
510   if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH &&
511       TM->getTargetTriple().isOSBinFormatMachO())
512     addPass(createAArch64CollectLOHPass());
513 }
514