1 //===- Parsing, selection, and construction of pass pipelines -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// 10 /// This file provides the implementation of the PassBuilder based on our 11 /// static pass registry as well as related functionality. It also provides 12 /// helpers to aid in analyzing, debugging, and testing passes and pass 13 /// pipelines. 14 /// 15 //===----------------------------------------------------------------------===// 16 17 #include "llvm/Passes/PassBuilder.h" 18 #include "llvm/ADT/StringSwitch.h" 19 #include "llvm/Analysis/AliasAnalysis.h" 20 #include "llvm/Analysis/AliasAnalysisEvaluator.h" 21 #include "llvm/Analysis/AssumptionCache.h" 22 #include "llvm/Analysis/BasicAliasAnalysis.h" 23 #include "llvm/Analysis/BlockFrequencyInfo.h" 24 #include "llvm/Analysis/BranchProbabilityInfo.h" 25 #include "llvm/Analysis/CFGPrinter.h" 26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h" 27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h" 28 #include "llvm/Analysis/CGSCCPassManager.h" 29 #include "llvm/Analysis/CallGraph.h" 30 #include "llvm/Analysis/DDG.h" 31 #include "llvm/Analysis/DemandedBits.h" 32 #include "llvm/Analysis/DependenceAnalysis.h" 33 #include "llvm/Analysis/DominanceFrontier.h" 34 #include "llvm/Analysis/GlobalsModRef.h" 35 #include "llvm/Analysis/IVUsers.h" 36 #include "llvm/Analysis/InlineAdvisor.h" 37 #include "llvm/Analysis/InlineFeaturesAnalysis.h" 38 #include "llvm/Analysis/LazyCallGraph.h" 39 #include "llvm/Analysis/LazyValueInfo.h" 40 #include "llvm/Analysis/LoopAccessAnalysis.h" 41 #include "llvm/Analysis/LoopCacheAnalysis.h" 42 #include "llvm/Analysis/LoopInfo.h" 43 #include "llvm/Analysis/LoopNestAnalysis.h" 44 #include "llvm/Analysis/MemoryDependenceAnalysis.h" 45 #include "llvm/Analysis/MemorySSA.h" 46 #include "llvm/Analysis/ModuleSummaryAnalysis.h" 47 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 48 #include "llvm/Analysis/PhiValues.h" 49 #include "llvm/Analysis/PostDominators.h" 50 #include "llvm/Analysis/ProfileSummaryInfo.h" 51 #include "llvm/Analysis/RegionInfo.h" 52 #include "llvm/Analysis/ScalarEvolution.h" 53 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" 54 #include "llvm/Analysis/ScopedNoAliasAA.h" 55 #include "llvm/Analysis/StackLifetime.h" 56 #include "llvm/Analysis/StackSafetyAnalysis.h" 57 #include "llvm/Analysis/TargetLibraryInfo.h" 58 #include "llvm/Analysis/TargetTransformInfo.h" 59 #include "llvm/Analysis/TypeBasedAliasAnalysis.h" 60 #include "llvm/IR/Dominators.h" 61 #include "llvm/IR/IRPrintingPasses.h" 62 #include "llvm/IR/PassManager.h" 63 #include "llvm/IR/SafepointIRVerifier.h" 64 #include "llvm/IR/Verifier.h" 65 #include "llvm/Support/CommandLine.h" 66 #include "llvm/Support/Debug.h" 67 #include "llvm/Support/FormatVariadic.h" 68 #include "llvm/Support/Regex.h" 69 #include "llvm/Target/TargetMachine.h" 70 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" 71 #include "llvm/Transforms/Coroutines/CoroCleanup.h" 72 #include "llvm/Transforms/Coroutines/CoroEarly.h" 73 #include "llvm/Transforms/Coroutines/CoroElide.h" 74 #include "llvm/Transforms/Coroutines/CoroSplit.h" 75 #include "llvm/Transforms/IPO/AlwaysInliner.h" 76 #include "llvm/Transforms/IPO/ArgumentPromotion.h" 77 #include "llvm/Transforms/IPO/Attributor.h" 78 #include "llvm/Transforms/IPO/CalledValuePropagation.h" 79 #include "llvm/Transforms/IPO/ConstantMerge.h" 80 #include "llvm/Transforms/IPO/CrossDSOCFI.h" 81 #include "llvm/Transforms/IPO/DeadArgumentElimination.h" 82 #include "llvm/Transforms/IPO/ElimAvailExtern.h" 83 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" 84 #include "llvm/Transforms/IPO/FunctionAttrs.h" 85 #include "llvm/Transforms/IPO/FunctionImport.h" 86 #include "llvm/Transforms/IPO/GlobalDCE.h" 87 #include "llvm/Transforms/IPO/GlobalOpt.h" 88 #include "llvm/Transforms/IPO/GlobalSplit.h" 89 #include "llvm/Transforms/IPO/HotColdSplitting.h" 90 #include "llvm/Transforms/IPO/InferFunctionAttrs.h" 91 #include "llvm/Transforms/IPO/Inliner.h" 92 #include "llvm/Transforms/IPO/Internalize.h" 93 #include "llvm/Transforms/IPO/LowerTypeTests.h" 94 #include "llvm/Transforms/IPO/MergeFunctions.h" 95 #include "llvm/Transforms/IPO/OpenMPOpt.h" 96 #include "llvm/Transforms/IPO/PartialInlining.h" 97 #include "llvm/Transforms/IPO/SCCP.h" 98 #include "llvm/Transforms/IPO/SampleProfile.h" 99 #include "llvm/Transforms/IPO/StripDeadPrototypes.h" 100 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" 101 #include "llvm/Transforms/IPO/WholeProgramDevirt.h" 102 #include "llvm/Transforms/InstCombine/InstCombine.h" 103 #include "llvm/Transforms/Instrumentation.h" 104 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 105 #include "llvm/Transforms/Instrumentation/BoundsChecking.h" 106 #include "llvm/Transforms/Instrumentation/CGProfile.h" 107 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h" 108 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" 109 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 110 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h" 111 #include "llvm/Transforms/Instrumentation/InstrProfiling.h" 112 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" 113 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" 114 #include "llvm/Transforms/Instrumentation/PoisonChecking.h" 115 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" 116 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 117 #include "llvm/Transforms/Scalar/ADCE.h" 118 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h" 119 #include "llvm/Transforms/Scalar/BDCE.h" 120 #include "llvm/Transforms/Scalar/CallSiteSplitting.h" 121 #include "llvm/Transforms/Scalar/ConstantHoisting.h" 122 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h" 123 #include "llvm/Transforms/Scalar/DCE.h" 124 #include "llvm/Transforms/Scalar/DeadStoreElimination.h" 125 #include "llvm/Transforms/Scalar/DivRemPairs.h" 126 #include "llvm/Transforms/Scalar/EarlyCSE.h" 127 #include "llvm/Transforms/Scalar/Float2Int.h" 128 #include "llvm/Transforms/Scalar/GVN.h" 129 #include "llvm/Transforms/Scalar/GuardWidening.h" 130 #include "llvm/Transforms/Scalar/IVUsersPrinter.h" 131 #include "llvm/Transforms/Scalar/IndVarSimplify.h" 132 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h" 133 #include "llvm/Transforms/Scalar/InstSimplifyPass.h" 134 #include "llvm/Transforms/Scalar/JumpThreading.h" 135 #include "llvm/Transforms/Scalar/LICM.h" 136 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h" 137 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h" 138 #include "llvm/Transforms/Scalar/LoopDeletion.h" 139 #include "llvm/Transforms/Scalar/LoopDistribute.h" 140 #include "llvm/Transforms/Scalar/LoopFuse.h" 141 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h" 142 #include "llvm/Transforms/Scalar/LoopInstSimplify.h" 143 #include "llvm/Transforms/Scalar/LoopLoadElimination.h" 144 #include "llvm/Transforms/Scalar/LoopPassManager.h" 145 #include "llvm/Transforms/Scalar/LoopPredication.h" 146 #include "llvm/Transforms/Scalar/LoopRotation.h" 147 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h" 148 #include "llvm/Transforms/Scalar/LoopSink.h" 149 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" 150 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" 151 #include "llvm/Transforms/Scalar/LoopUnrollPass.h" 152 #include "llvm/Transforms/Scalar/LowerAtomic.h" 153 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" 154 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" 155 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h" 156 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h" 157 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h" 158 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h" 159 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" 160 #include "llvm/Transforms/Scalar/MergeICmps.h" 161 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" 162 #include "llvm/Transforms/Scalar/NaryReassociate.h" 163 #include "llvm/Transforms/Scalar/NewGVN.h" 164 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" 165 #include "llvm/Transforms/Scalar/Reassociate.h" 166 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h" 167 #include "llvm/Transforms/Scalar/SCCP.h" 168 #include "llvm/Transforms/Scalar/SROA.h" 169 #include "llvm/Transforms/Scalar/Scalarizer.h" 170 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" 171 #include "llvm/Transforms/Scalar/SimplifyCFG.h" 172 #include "llvm/Transforms/Scalar/Sink.h" 173 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h" 174 #include "llvm/Transforms/Scalar/SpeculativeExecution.h" 175 #include "llvm/Transforms/Scalar/TailRecursionElimination.h" 176 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" 177 #include "llvm/Transforms/Utils/AddDiscriminators.h" 178 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" 179 #include "llvm/Transforms/Utils/BreakCriticalEdges.h" 180 #include "llvm/Transforms/Utils/CanonicalizeAliases.h" 181 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" 182 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" 183 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 184 #include "llvm/Transforms/Utils/LCSSA.h" 185 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" 186 #include "llvm/Transforms/Utils/LoopSimplify.h" 187 #include "llvm/Transforms/Utils/LowerInvoke.h" 188 #include "llvm/Transforms/Utils/Mem2Reg.h" 189 #include "llvm/Transforms/Utils/NameAnonGlobals.h" 190 #include "llvm/Transforms/Utils/SymbolRewriter.h" 191 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h" 192 #include "llvm/Transforms/Vectorize/LoopVectorize.h" 193 #include "llvm/Transforms/Vectorize/SLPVectorizer.h" 194 #include "llvm/Transforms/Vectorize/VectorCombine.h" 195 196 using namespace llvm; 197 198 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations", 199 cl::ReallyHidden, cl::init(4)); 200 static cl::opt<bool> 201 RunPartialInlining("enable-npm-partial-inlining", cl::init(false), 202 cl::Hidden, cl::ZeroOrMore, 203 cl::desc("Run Partial inlinining pass")); 204 205 static cl::opt<int> PreInlineThreshold( 206 "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore, 207 cl::desc("Control the amount of inlining in pre-instrumentation inliner " 208 "(default = 75)")); 209 210 static cl::opt<bool> 211 RunNewGVN("enable-npm-newgvn", cl::init(false), 212 cl::Hidden, cl::ZeroOrMore, 213 cl::desc("Run NewGVN instead of GVN")); 214 215 static cl::opt<bool> EnableGVNHoist( 216 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden, 217 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); 218 219 static cl::opt<InliningAdvisorMode> UseInlineAdvisor( 220 "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, 221 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), 222 cl::values(clEnumValN(InliningAdvisorMode::Default, "default", 223 "Heuristics-based inliner version."), 224 clEnumValN(InliningAdvisorMode::Development, "development", 225 "Use development mode (runtime-loadable model)."), 226 clEnumValN(InliningAdvisorMode::Release, "release", 227 "Use release mode (AOT-compiled model)."))); 228 229 static cl::opt<bool> EnableGVNSink( 230 "enable-npm-gvn-sink", cl::init(false), cl::Hidden, 231 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); 232 233 static cl::opt<bool> EnableUnrollAndJam( 234 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden, 235 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)")); 236 237 static cl::opt<bool> EnableSyntheticCounts( 238 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, 239 cl::desc("Run synthetic function entry count generation " 240 "pass")); 241 242 static const Regex DefaultAliasRegex( 243 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$"); 244 245 // This option is used in simplifying testing SampleFDO optimizations for 246 // profile loading. 247 static cl::opt<bool> 248 EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden, 249 cl::desc("Enable control height reduction optimization (CHR)")); 250 251 /// Flag to enable inline deferral during PGO. 252 static cl::opt<bool> 253 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), 254 cl::Hidden, 255 cl::desc("Enable inline deferral during PGO")); 256 257 PipelineTuningOptions::PipelineTuningOptions() { 258 LoopInterleaving = true; 259 LoopVectorization = true; 260 SLPVectorization = false; 261 LoopUnrolling = true; 262 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; 263 Coroutines = false; 264 LicmMssaOptCap = SetLicmMssaOptCap; 265 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; 266 CallGraphProfile = true; 267 } 268 269 extern cl::opt<bool> EnableHotColdSplit; 270 extern cl::opt<bool> EnableOrderFileInstrumentation; 271 272 extern cl::opt<bool> FlattenedProfileUsed; 273 274 extern cl::opt<AttributorRunOption> AttributorRun; 275 extern cl::opt<bool> EnableKnowledgeRetention; 276 277 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = { 278 /*SpeedLevel*/ 0, 279 /*SizeLevel*/ 0}; 280 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = { 281 /*SpeedLevel*/ 1, 282 /*SizeLevel*/ 0}; 283 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = { 284 /*SpeedLevel*/ 2, 285 /*SizeLevel*/ 0}; 286 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = { 287 /*SpeedLevel*/ 3, 288 /*SizeLevel*/ 0}; 289 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = { 290 /*SpeedLevel*/ 2, 291 /*SizeLevel*/ 1}; 292 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = { 293 /*SpeedLevel*/ 2, 294 /*SizeLevel*/ 2}; 295 296 namespace { 297 298 // The following passes/analyses have custom names, otherwise their name will 299 // include `(anonymous namespace)`. These are special since they are only for 300 // testing purposes and don't live in a header file. 301 302 /// No-op module pass which does nothing. 303 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> { 304 PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { 305 return PreservedAnalyses::all(); 306 } 307 308 static StringRef name() { return "NoOpModulePass"; } 309 }; 310 311 /// No-op module analysis. 312 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> { 313 friend AnalysisInfoMixin<NoOpModuleAnalysis>; 314 static AnalysisKey Key; 315 316 public: 317 struct Result {}; 318 Result run(Module &, ModuleAnalysisManager &) { return Result(); } 319 static StringRef name() { return "NoOpModuleAnalysis"; } 320 }; 321 322 /// No-op CGSCC pass which does nothing. 323 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> { 324 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &, 325 LazyCallGraph &, CGSCCUpdateResult &UR) { 326 return PreservedAnalyses::all(); 327 } 328 static StringRef name() { return "NoOpCGSCCPass"; } 329 }; 330 331 /// No-op CGSCC analysis. 332 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> { 333 friend AnalysisInfoMixin<NoOpCGSCCAnalysis>; 334 static AnalysisKey Key; 335 336 public: 337 struct Result {}; 338 Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) { 339 return Result(); 340 } 341 static StringRef name() { return "NoOpCGSCCAnalysis"; } 342 }; 343 344 /// No-op function pass which does nothing. 345 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> { 346 PreservedAnalyses run(Function &F, FunctionAnalysisManager &) { 347 return PreservedAnalyses::all(); 348 } 349 static StringRef name() { return "NoOpFunctionPass"; } 350 }; 351 352 /// No-op function analysis. 353 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> { 354 friend AnalysisInfoMixin<NoOpFunctionAnalysis>; 355 static AnalysisKey Key; 356 357 public: 358 struct Result {}; 359 Result run(Function &, FunctionAnalysisManager &) { return Result(); } 360 static StringRef name() { return "NoOpFunctionAnalysis"; } 361 }; 362 363 /// No-op loop pass which does nothing. 364 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> { 365 PreservedAnalyses run(Loop &L, LoopAnalysisManager &, 366 LoopStandardAnalysisResults &, LPMUpdater &) { 367 return PreservedAnalyses::all(); 368 } 369 static StringRef name() { return "NoOpLoopPass"; } 370 }; 371 372 /// No-op loop analysis. 373 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> { 374 friend AnalysisInfoMixin<NoOpLoopAnalysis>; 375 static AnalysisKey Key; 376 377 public: 378 struct Result {}; 379 Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) { 380 return Result(); 381 } 382 static StringRef name() { return "NoOpLoopAnalysis"; } 383 }; 384 385 AnalysisKey NoOpModuleAnalysis::Key; 386 AnalysisKey NoOpCGSCCAnalysis::Key; 387 AnalysisKey NoOpFunctionAnalysis::Key; 388 AnalysisKey NoOpLoopAnalysis::Key; 389 390 } // namespace 391 392 void PassBuilder::invokePeepholeEPCallbacks( 393 FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 394 for (auto &C : PeepholeEPCallbacks) 395 C(FPM, Level); 396 } 397 398 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { 399 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 400 MAM.registerPass([&] { return CREATE_PASS; }); 401 #include "PassRegistry.def" 402 403 for (auto &C : ModuleAnalysisRegistrationCallbacks) 404 C(MAM); 405 } 406 407 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { 408 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 409 CGAM.registerPass([&] { return CREATE_PASS; }); 410 #include "PassRegistry.def" 411 412 for (auto &C : CGSCCAnalysisRegistrationCallbacks) 413 C(CGAM); 414 } 415 416 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { 417 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 418 FAM.registerPass([&] { return CREATE_PASS; }); 419 #include "PassRegistry.def" 420 421 for (auto &C : FunctionAnalysisRegistrationCallbacks) 422 C(FAM); 423 } 424 425 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) { 426 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 427 LAM.registerPass([&] { return CREATE_PASS; }); 428 #include "PassRegistry.def" 429 430 for (auto &C : LoopAnalysisRegistrationCallbacks) 431 C(LAM); 432 } 433 434 // TODO: Investigate the cost/benefit of tail call elimination on debugging. 435 FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline( 436 OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { 437 438 FunctionPassManager FPM(DebugLogging); 439 440 // Form SSA out of local memory accesses after breaking apart aggregates into 441 // scalars. 442 FPM.addPass(SROA()); 443 444 // Catch trivial redundancies 445 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); 446 447 // Hoisting of scalars and load expressions. 448 FPM.addPass(SimplifyCFGPass()); 449 FPM.addPass(InstCombinePass()); 450 451 FPM.addPass(LibCallsShrinkWrapPass()); 452 453 invokePeepholeEPCallbacks(FPM, Level); 454 455 FPM.addPass(SimplifyCFGPass()); 456 457 // Form canonically associated expression trees, and simplify the trees using 458 // basic mathematical properties. For example, this will form (nearly) 459 // minimal multiplication trees. 460 FPM.addPass(ReassociatePass()); 461 462 // Add the primary loop simplification pipeline. 463 // FIXME: Currently this is split into two loop pass pipelines because we run 464 // some function passes in between them. These can and should be removed 465 // and/or replaced by scheduling the loop pass equivalents in the correct 466 // positions. But those equivalent passes aren't powerful enough yet. 467 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still 468 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to 469 // fully replace `SimplifyCFGPass`, and the closest to the other we have is 470 // `LoopInstSimplify`. 471 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); 472 473 // Simplify the loop body. We do this initially to clean up after other loop 474 // passes run, either when iterating on a loop or on inner loops with 475 // implications on the outer loop. 476 LPM1.addPass(LoopInstSimplifyPass()); 477 LPM1.addPass(LoopSimplifyCFGPass()); 478 479 LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true)); 480 // TODO: Investigate promotion cap for O1. 481 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap)); 482 LPM1.addPass(SimpleLoopUnswitchPass()); 483 LPM2.addPass(IndVarSimplifyPass()); 484 LPM2.addPass(LoopIdiomRecognizePass()); 485 486 for (auto &C : LateLoopOptimizationsEPCallbacks) 487 C(LPM2, Level); 488 489 LPM2.addPass(LoopDeletionPass()); 490 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO 491 // because it changes IR to makes profile annotation in back compile 492 // inaccurate. The normal unroller doesn't pay attention to forced full unroll 493 // attributes so we need to make sure and allow the full unroll pass to pay 494 // attention to it. 495 if (Phase != ThinLTOPhase::PreLink || !PGOOpt || 496 PGOOpt->Action != PGOOptions::SampleUse) 497 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), 498 /* OnlyWhenForced= */ !PTO.LoopUnrolling, 499 PTO.ForgetAllSCEVInLoopUnroll)); 500 501 for (auto &C : LoopOptimizerEndEPCallbacks) 502 C(LPM2, Level); 503 504 // We provide the opt remark emitter pass for LICM to use. We only need to do 505 // this once as it is immutable. 506 FPM.addPass( 507 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); 508 FPM.addPass(createFunctionToLoopPassAdaptor( 509 std::move(LPM1), EnableMSSALoopDependency, DebugLogging)); 510 FPM.addPass(SimplifyCFGPass()); 511 FPM.addPass(InstCombinePass()); 512 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA. 513 // *All* loop passes must preserve it, in order to be able to use it. 514 FPM.addPass(createFunctionToLoopPassAdaptor( 515 std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging)); 516 517 // Delete small array after loop unroll. 518 FPM.addPass(SROA()); 519 520 // Specially optimize memory movement as it doesn't look like dataflow in SSA. 521 FPM.addPass(MemCpyOptPass()); 522 523 // Sparse conditional constant propagation. 524 // FIXME: It isn't clear why we do this *after* loop passes rather than 525 // before... 526 FPM.addPass(SCCPPass()); 527 528 // Delete dead bit computations (instcombine runs after to fold away the dead 529 // computations, and then ADCE will run later to exploit any new DCE 530 // opportunities that creates). 531 FPM.addPass(BDCEPass()); 532 533 // Run instcombine after redundancy and dead bit elimination to exploit 534 // opportunities opened up by them. 535 FPM.addPass(InstCombinePass()); 536 invokePeepholeEPCallbacks(FPM, Level); 537 538 if (PTO.Coroutines) 539 FPM.addPass(CoroElidePass()); 540 541 for (auto &C : ScalarOptimizerLateEPCallbacks) 542 C(FPM, Level); 543 544 // Finally, do an expensive DCE pass to catch all the dead code exposed by 545 // the simplifications and basic cleanup after all the simplifications. 546 // TODO: Investigate if this is too expensive. 547 FPM.addPass(ADCEPass()); 548 FPM.addPass(SimplifyCFGPass()); 549 FPM.addPass(InstCombinePass()); 550 invokePeepholeEPCallbacks(FPM, Level); 551 552 return FPM; 553 } 554 555 FunctionPassManager 556 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, 557 ThinLTOPhase Phase, 558 bool DebugLogging) { 559 assert(Level != OptimizationLevel::O0 && "Must request optimizations!"); 560 561 // The O1 pipeline has a separate pipeline creation function to simplify 562 // construction readability. 563 if (Level.getSpeedupLevel() == 1) 564 return buildO1FunctionSimplificationPipeline(Level, Phase, DebugLogging); 565 566 FunctionPassManager FPM(DebugLogging); 567 568 // Form SSA out of local memory accesses after breaking apart aggregates into 569 // scalars. 570 FPM.addPass(SROA()); 571 572 // Catch trivial redundancies 573 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); 574 if (EnableKnowledgeRetention) 575 FPM.addPass(AssumeSimplifyPass()); 576 577 // Hoisting of scalars and load expressions. 578 if (EnableGVNHoist) 579 FPM.addPass(GVNHoistPass()); 580 581 // Global value numbering based sinking. 582 if (EnableGVNSink) { 583 FPM.addPass(GVNSinkPass()); 584 FPM.addPass(SimplifyCFGPass()); 585 } 586 587 // Speculative execution if the target has divergent branches; otherwise nop. 588 FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true)); 589 590 // Optimize based on known information about branches, and cleanup afterward. 591 FPM.addPass(JumpThreadingPass()); 592 FPM.addPass(CorrelatedValuePropagationPass()); 593 594 FPM.addPass(SimplifyCFGPass()); 595 if (Level == OptimizationLevel::O3) 596 FPM.addPass(AggressiveInstCombinePass()); 597 FPM.addPass(InstCombinePass()); 598 599 if (!Level.isOptimizingForSize()) 600 FPM.addPass(LibCallsShrinkWrapPass()); 601 602 invokePeepholeEPCallbacks(FPM, Level); 603 604 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy 605 // using the size value profile. Don't perform this when optimizing for size. 606 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse && 607 !Level.isOptimizingForSize()) 608 FPM.addPass(PGOMemOPSizeOpt()); 609 610 FPM.addPass(TailCallElimPass()); 611 FPM.addPass(SimplifyCFGPass()); 612 613 // Form canonically associated expression trees, and simplify the trees using 614 // basic mathematical properties. For example, this will form (nearly) 615 // minimal multiplication trees. 616 FPM.addPass(ReassociatePass()); 617 618 // Add the primary loop simplification pipeline. 619 // FIXME: Currently this is split into two loop pass pipelines because we run 620 // some function passes in between them. These can and should be removed 621 // and/or replaced by scheduling the loop pass equivalents in the correct 622 // positions. But those equivalent passes aren't powerful enough yet. 623 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still 624 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to 625 // fully replace `SimplifyCFGPass`, and the closest to the other we have is 626 // `LoopInstSimplify`. 627 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); 628 629 // Simplify the loop body. We do this initially to clean up after other loop 630 // passes run, either when iterating on a loop or on inner loops with 631 // implications on the outer loop. 632 LPM1.addPass(LoopInstSimplifyPass()); 633 LPM1.addPass(LoopSimplifyCFGPass()); 634 635 // Rotate Loop - disable header duplication at -Oz 636 LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz)); 637 // TODO: Investigate promotion cap for O1. 638 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap)); 639 LPM1.addPass(SimpleLoopUnswitchPass()); 640 LPM2.addPass(IndVarSimplifyPass()); 641 LPM2.addPass(LoopIdiomRecognizePass()); 642 643 for (auto &C : LateLoopOptimizationsEPCallbacks) 644 C(LPM2, Level); 645 646 LPM2.addPass(LoopDeletionPass()); 647 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO 648 // because it changes IR to makes profile annotation in back compile 649 // inaccurate. The normal unroller doesn't pay attention to forced full unroll 650 // attributes so we need to make sure and allow the full unroll pass to pay 651 // attention to it. 652 if (Phase != ThinLTOPhase::PreLink || !PGOOpt || 653 PGOOpt->Action != PGOOptions::SampleUse) 654 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), 655 /* OnlyWhenForced= */ !PTO.LoopUnrolling, 656 PTO.ForgetAllSCEVInLoopUnroll)); 657 658 for (auto &C : LoopOptimizerEndEPCallbacks) 659 C(LPM2, Level); 660 661 // We provide the opt remark emitter pass for LICM to use. We only need to do 662 // this once as it is immutable. 663 FPM.addPass( 664 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); 665 FPM.addPass(createFunctionToLoopPassAdaptor( 666 std::move(LPM1), EnableMSSALoopDependency, DebugLogging)); 667 FPM.addPass(SimplifyCFGPass()); 668 FPM.addPass(InstCombinePass()); 669 // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass, 670 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA. 671 // *All* loop passes must preserve it, in order to be able to use it. 672 FPM.addPass(createFunctionToLoopPassAdaptor( 673 std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging)); 674 675 // Delete small array after loop unroll. 676 FPM.addPass(SROA()); 677 678 // Eliminate redundancies. 679 FPM.addPass(MergedLoadStoreMotionPass()); 680 if (RunNewGVN) 681 FPM.addPass(NewGVNPass()); 682 else 683 FPM.addPass(GVN()); 684 685 // Specially optimize memory movement as it doesn't look like dataflow in SSA. 686 FPM.addPass(MemCpyOptPass()); 687 688 // Sparse conditional constant propagation. 689 // FIXME: It isn't clear why we do this *after* loop passes rather than 690 // before... 691 FPM.addPass(SCCPPass()); 692 693 // Delete dead bit computations (instcombine runs after to fold away the dead 694 // computations, and then ADCE will run later to exploit any new DCE 695 // opportunities that creates). 696 FPM.addPass(BDCEPass()); 697 698 // Run instcombine after redundancy and dead bit elimination to exploit 699 // opportunities opened up by them. 700 FPM.addPass(InstCombinePass()); 701 invokePeepholeEPCallbacks(FPM, Level); 702 703 // Re-consider control flow based optimizations after redundancy elimination, 704 // redo DCE, etc. 705 FPM.addPass(JumpThreadingPass()); 706 FPM.addPass(CorrelatedValuePropagationPass()); 707 FPM.addPass(DSEPass()); 708 FPM.addPass(createFunctionToLoopPassAdaptor( 709 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), 710 EnableMSSALoopDependency, DebugLogging)); 711 712 if (PTO.Coroutines) 713 FPM.addPass(CoroElidePass()); 714 715 for (auto &C : ScalarOptimizerLateEPCallbacks) 716 C(FPM, Level); 717 718 // Finally, do an expensive DCE pass to catch all the dead code exposed by 719 // the simplifications and basic cleanup after all the simplifications. 720 // TODO: Investigate if this is too expensive. 721 FPM.addPass(ADCEPass()); 722 FPM.addPass(SimplifyCFGPass()); 723 FPM.addPass(InstCombinePass()); 724 invokePeepholeEPCallbacks(FPM, Level); 725 726 if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt && 727 (PGOOpt->Action == PGOOptions::IRUse || 728 PGOOpt->Action == PGOOptions::SampleUse)) 729 FPM.addPass(ControlHeightReductionPass()); 730 731 return FPM; 732 } 733 734 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, 735 PassBuilder::OptimizationLevel Level, 736 bool RunProfileGen, bool IsCS, 737 std::string ProfileFile, 738 std::string ProfileRemappingFile) { 739 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!"); 740 // Generally running simplification passes and the inliner with an high 741 // threshold results in smaller executables, but there may be cases where 742 // the size grows, so let's be conservative here and skip this simplification 743 // at -Os/Oz. We will not do this inline for context sensistive PGO (when 744 // IsCS is true). 745 if (!Level.isOptimizingForSize() && !IsCS) { 746 InlineParams IP; 747 748 IP.DefaultThreshold = PreInlineThreshold; 749 750 // FIXME: The hint threshold has the same value used by the regular inliner. 751 // This should probably be lowered after performance testing. 752 // FIXME: this comment is cargo culted from the old pass manager, revisit). 753 IP.HintThreshold = 325; 754 ModuleInlinerWrapperPass MIWP(IP, DebugLogging); 755 CGSCCPassManager &CGPipeline = MIWP.getPM(); 756 757 FunctionPassManager FPM; 758 FPM.addPass(SROA()); 759 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. 760 FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. 761 FPM.addPass(InstCombinePass()); // Combine silly sequences. 762 invokePeepholeEPCallbacks(FPM, Level); 763 764 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); 765 766 MPM.addPass(std::move(MIWP)); 767 768 // Delete anything that is now dead to make sure that we don't instrument 769 // dead code. Instrumentation can end up keeping dead code around and 770 // dramatically increase code size. 771 MPM.addPass(GlobalDCEPass()); 772 } 773 774 if (!RunProfileGen) { 775 assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); 776 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS)); 777 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert 778 // RequireAnalysisPass for PSI before subsequent non-module passes. 779 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); 780 return; 781 } 782 783 // Perform PGO instrumentation. 784 MPM.addPass(PGOInstrumentationGen(IsCS)); 785 786 FunctionPassManager FPM; 787 FPM.addPass(createFunctionToLoopPassAdaptor( 788 LoopRotatePass(), EnableMSSALoopDependency, DebugLogging)); 789 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 790 791 // Add the profile lowering pass. 792 InstrProfOptions Options; 793 if (!ProfileFile.empty()) 794 Options.InstrProfileOutput = ProfileFile; 795 // Do counter promotion at Level greater than O0. 796 Options.DoCounterPromotion = true; 797 Options.UseBFIInPromotion = IsCS; 798 MPM.addPass(InstrProfiling(Options, IsCS)); 799 } 800 801 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM, 802 bool DebugLogging, bool RunProfileGen, 803 bool IsCS, std::string ProfileFile, 804 std::string ProfileRemappingFile) { 805 if (!RunProfileGen) { 806 assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); 807 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS)); 808 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert 809 // RequireAnalysisPass for PSI before subsequent non-module passes. 810 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); 811 return; 812 } 813 814 // Perform PGO instrumentation. 815 MPM.addPass(PGOInstrumentationGen(IsCS)); 816 // Add the profile lowering pass. 817 InstrProfOptions Options; 818 if (!ProfileFile.empty()) 819 Options.InstrProfileOutput = ProfileFile; 820 // Do not do counter promotion at O0. 821 Options.DoCounterPromotion = false; 822 Options.UseBFIInPromotion = IsCS; 823 MPM.addPass(InstrProfiling(Options, IsCS)); 824 } 825 826 static InlineParams 827 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { 828 return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel()); 829 } 830 831 ModuleInlinerWrapperPass 832 PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase, 833 bool DebugLogging) { 834 InlineParams IP = getInlineParamsFromOptLevel(Level); 835 if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt && 836 PGOOpt->Action == PGOOptions::SampleUse) 837 IP.HotCallSiteThreshold = 0; 838 839 if (PGOOpt) 840 IP.EnableDeferral = EnablePGOInlineDeferral; 841 842 ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor, 843 MaxDevirtIterations); 844 845 // Require the GlobalsAA analysis for the module so we can query it within 846 // the CGSCC pipeline. 847 MIWP.addRequiredModuleAnalysis<GlobalsAA>(); 848 849 // Require the ProfileSummaryAnalysis for the module so we can query it within 850 // the inliner pass. 851 MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>(); 852 853 // Now begin the main postorder CGSCC pipeline. 854 // FIXME: The current CGSCC pipeline has its origins in the legacy pass 855 // manager and trying to emulate its precise behavior. Much of this doesn't 856 // make a lot of sense and we should revisit the core CGSCC structure. 857 CGSCCPassManager &MainCGPipeline = MIWP.getPM(); 858 859 // Note: historically, the PruneEH pass was run first to deduce nounwind and 860 // generally clean up exception handling overhead. It isn't clear this is 861 // valuable as the inliner doesn't currently care whether it is inlining an 862 // invoke or a call. 863 864 if (AttributorRun & AttributorRunOption::CGSCC) 865 MainCGPipeline.addPass(AttributorCGSCCPass()); 866 867 if (PTO.Coroutines) 868 MainCGPipeline.addPass(CoroSplitPass()); 869 870 // Now deduce any function attributes based in the current code. 871 MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); 872 873 // When at O3 add argument promotion to the pass pipeline. 874 // FIXME: It isn't at all clear why this should be limited to O3. 875 if (Level == OptimizationLevel::O3) 876 MainCGPipeline.addPass(ArgumentPromotionPass()); 877 878 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if 879 // there are no OpenMP runtime calls present in the module. 880 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3) 881 MainCGPipeline.addPass(OpenMPOptPass()); 882 883 // Lastly, add the core function simplification pipeline nested inside the 884 // CGSCC walk. 885 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( 886 buildFunctionSimplificationPipeline(Level, Phase, DebugLogging))); 887 888 for (auto &C : CGSCCOptimizerLateEPCallbacks) 889 C(MainCGPipeline, Level); 890 891 return MIWP; 892 } 893 894 ModulePassManager PassBuilder::buildModuleSimplificationPipeline( 895 OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { 896 ModulePassManager MPM(DebugLogging); 897 898 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse); 899 900 // In ThinLTO mode, when flattened profile is used, all the available 901 // profile information will be annotated in PreLink phase so there is 902 // no need to load the profile again in PostLink. 903 bool LoadSampleProfile = 904 HasSampleProfile && 905 !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink); 906 907 // During the ThinLTO backend phase we perform early indirect call promotion 908 // here, before globalopt. Otherwise imported available_externally functions 909 // look unreferenced and are removed. If we are going to load the sample 910 // profile then defer until later. 911 // TODO: See if we can move later and consolidate with the location where 912 // we perform ICP when we are loading a sample profile. 913 // TODO: We pass HasSampleProfile (whether there was a sample profile file 914 // passed to the compile) to the SamplePGO flag of ICP. This is used to 915 // determine whether the new direct calls are annotated with prof metadata. 916 // Ideally this should be determined from whether the IR is annotated with 917 // sample profile, and not whether the a sample profile was provided on the 918 // command line. E.g. for flattened profiles where we will not be reloading 919 // the sample profile in the ThinLTO backend, we ideally shouldn't have to 920 // provide the sample profile file. 921 if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile) 922 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile)); 923 924 // Do basic inference of function attributes from known properties of system 925 // libraries and other oracles. 926 MPM.addPass(InferFunctionAttrsPass()); 927 928 // Create an early function pass manager to cleanup the output of the 929 // frontend. 930 FunctionPassManager EarlyFPM(DebugLogging); 931 EarlyFPM.addPass(SimplifyCFGPass()); 932 EarlyFPM.addPass(SROA()); 933 EarlyFPM.addPass(EarlyCSEPass()); 934 EarlyFPM.addPass(LowerExpectIntrinsicPass()); 935 if (PTO.Coroutines) 936 EarlyFPM.addPass(CoroEarlyPass()); 937 if (Level == OptimizationLevel::O3) 938 EarlyFPM.addPass(CallSiteSplittingPass()); 939 940 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation 941 // to convert bitcast to direct calls so that they can be inlined during the 942 // profile annotation prepration step. 943 // More details about SamplePGO design can be found in: 944 // https://research.google.com/pubs/pub45290.html 945 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. 946 if (LoadSampleProfile) 947 EarlyFPM.addPass(InstCombinePass()); 948 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); 949 950 if (LoadSampleProfile) { 951 // Annotate sample profile right after early FPM to ensure freshness of 952 // the debug info. 953 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, 954 PGOOpt->ProfileRemappingFile, 955 Phase == ThinLTOPhase::PreLink)); 956 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert 957 // RequireAnalysisPass for PSI before subsequent non-module passes. 958 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); 959 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard 960 // for the profile annotation to be accurate in the ThinLTO backend. 961 if (Phase != ThinLTOPhase::PreLink) 962 // We perform early indirect call promotion here, before globalopt. 963 // This is important for the ThinLTO backend phase because otherwise 964 // imported available_externally functions look unreferenced and are 965 // removed. 966 MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, 967 true /* SamplePGO */)); 968 } 969 970 if (AttributorRun & AttributorRunOption::MODULE) 971 MPM.addPass(AttributorPass()); 972 973 // Interprocedural constant propagation now that basic cleanup has occurred 974 // and prior to optimizing globals. 975 // FIXME: This position in the pipeline hasn't been carefully considered in 976 // years, it should be re-analyzed. 977 MPM.addPass(IPSCCPPass()); 978 979 // Attach metadata to indirect call sites indicating the set of functions 980 // they may target at run-time. This should follow IPSCCP. 981 MPM.addPass(CalledValuePropagationPass()); 982 983 // Optimize globals to try and fold them into constants. 984 MPM.addPass(GlobalOptPass()); 985 986 // Promote any localized globals to SSA registers. 987 // FIXME: Should this instead by a run of SROA? 988 // FIXME: We should probably run instcombine and simplify-cfg afterward to 989 // delete control flows that are dead once globals have been folded to 990 // constants. 991 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); 992 993 // Remove any dead arguments exposed by cleanups and constant folding 994 // globals. 995 MPM.addPass(DeadArgumentEliminationPass()); 996 997 // Create a small function pass pipeline to cleanup after all the global 998 // optimizations. 999 FunctionPassManager GlobalCleanupPM(DebugLogging); 1000 GlobalCleanupPM.addPass(InstCombinePass()); 1001 invokePeepholeEPCallbacks(GlobalCleanupPM, Level); 1002 1003 GlobalCleanupPM.addPass(SimplifyCFGPass()); 1004 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); 1005 1006 // Add all the requested passes for instrumentation PGO, if requested. 1007 if (PGOOpt && Phase != ThinLTOPhase::PostLink && 1008 (PGOOpt->Action == PGOOptions::IRInstr || 1009 PGOOpt->Action == PGOOptions::IRUse)) { 1010 addPGOInstrPasses(MPM, DebugLogging, Level, 1011 /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr, 1012 /* IsCS */ false, PGOOpt->ProfileFile, 1013 PGOOpt->ProfileRemappingFile); 1014 MPM.addPass(PGOIndirectCallPromotion(false, false)); 1015 } 1016 if (PGOOpt && Phase != ThinLTOPhase::PostLink && 1017 PGOOpt->CSAction == PGOOptions::CSIRInstr) 1018 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile)); 1019 1020 // Synthesize function entry counts for non-PGO compilation. 1021 if (EnableSyntheticCounts && !PGOOpt) 1022 MPM.addPass(SyntheticCountsPropagation()); 1023 1024 MPM.addPass(buildInlinerPipeline(Level, Phase, DebugLogging)); 1025 return MPM; 1026 } 1027 1028 ModulePassManager PassBuilder::buildModuleOptimizationPipeline( 1029 OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) { 1030 ModulePassManager MPM(DebugLogging); 1031 1032 // Optimize globals now that the module is fully simplified. 1033 MPM.addPass(GlobalOptPass()); 1034 MPM.addPass(GlobalDCEPass()); 1035 1036 // Run partial inlining pass to partially inline functions that have 1037 // large bodies. 1038 if (RunPartialInlining) 1039 MPM.addPass(PartialInlinerPass()); 1040 1041 // Remove avail extern fns and globals definitions since we aren't compiling 1042 // an object file for later LTO. For LTO we want to preserve these so they 1043 // are eligible for inlining at link-time. Note if they are unreferenced they 1044 // will be removed by GlobalDCE later, so this only impacts referenced 1045 // available externally globals. Eventually they will be suppressed during 1046 // codegen, but eliminating here enables more opportunity for GlobalDCE as it 1047 // may make globals referenced by available external functions dead and saves 1048 // running remaining passes on the eliminated functions. These should be 1049 // preserved during prelinking for link-time inlining decisions. 1050 if (!LTOPreLink) 1051 MPM.addPass(EliminateAvailableExternallyPass()); 1052 1053 if (EnableOrderFileInstrumentation) 1054 MPM.addPass(InstrOrderFilePass()); 1055 1056 // Do RPO function attribute inference across the module to forward-propagate 1057 // attributes where applicable. 1058 // FIXME: Is this really an optimization rather than a canonicalization? 1059 MPM.addPass(ReversePostOrderFunctionAttrsPass()); 1060 1061 // Do a post inline PGO instrumentation and use pass. This is a context 1062 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as 1063 // cross-module inline has not been done yet. The context sensitive 1064 // instrumentation is after all the inlines are done. 1065 if (!LTOPreLink && PGOOpt) { 1066 if (PGOOpt->CSAction == PGOOptions::CSIRInstr) 1067 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true, 1068 /* IsCS */ true, PGOOpt->CSProfileGenFile, 1069 PGOOpt->ProfileRemappingFile); 1070 else if (PGOOpt->CSAction == PGOOptions::CSIRUse) 1071 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false, 1072 /* IsCS */ true, PGOOpt->ProfileFile, 1073 PGOOpt->ProfileRemappingFile); 1074 } 1075 1076 // Re-require GloblasAA here prior to function passes. This is particularly 1077 // useful as the above will have inlined, DCE'ed, and function-attr 1078 // propagated everything. We should at this point have a reasonably minimal 1079 // and richly annotated call graph. By computing aliasing and mod/ref 1080 // information for all local globals here, the late loop passes and notably 1081 // the vectorizer will be able to use them to help recognize vectorizable 1082 // memory operations. 1083 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>()); 1084 1085 FunctionPassManager OptimizePM(DebugLogging); 1086 OptimizePM.addPass(Float2IntPass()); 1087 OptimizePM.addPass(LowerConstantIntrinsicsPass()); 1088 1089 // FIXME: We need to run some loop optimizations to re-rotate loops after 1090 // simplify-cfg and others undo their rotation. 1091 1092 // Optimize the loop execution. These passes operate on entire loop nests 1093 // rather than on each loop in an inside-out manner, and so they are actually 1094 // function passes. 1095 1096 for (auto &C : VectorizerStartEPCallbacks) 1097 C(OptimizePM, Level); 1098 1099 // First rotate loops that may have been un-rotated by prior passes. 1100 OptimizePM.addPass(createFunctionToLoopPassAdaptor( 1101 LoopRotatePass(), EnableMSSALoopDependency, DebugLogging)); 1102 1103 // Distribute loops to allow partial vectorization. I.e. isolate dependences 1104 // into separate loop that would otherwise inhibit vectorization. This is 1105 // currently only performed for loops marked with the metadata 1106 // llvm.loop.distribute=true or when -enable-loop-distribute is specified. 1107 OptimizePM.addPass(LoopDistributePass()); 1108 1109 // Populates the VFABI attribute with the scalar-to-vector mappings 1110 // from the TargetLibraryInfo. 1111 OptimizePM.addPass(InjectTLIMappings()); 1112 1113 // Now run the core loop vectorizer. 1114 OptimizePM.addPass(LoopVectorizePass( 1115 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization))); 1116 1117 // Eliminate loads by forwarding stores from the previous iteration to loads 1118 // of the current iteration. 1119 OptimizePM.addPass(LoopLoadEliminationPass()); 1120 1121 // Cleanup after the loop optimization passes. 1122 OptimizePM.addPass(InstCombinePass()); 1123 1124 // Now that we've formed fast to execute loop structures, we do further 1125 // optimizations. These are run afterward as they might block doing complex 1126 // analyses and transforms such as what are needed for loop vectorization. 1127 1128 // Cleanup after loop vectorization, etc. Simplification passes like CVP and 1129 // GVN, loop transforms, and others have already run, so it's now better to 1130 // convert to more optimized IR using more aggressive simplify CFG options. 1131 // The extra sinking transform can create larger basic blocks, so do this 1132 // before SLP vectorization. 1133 OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions(). 1134 forwardSwitchCondToPhi(true). 1135 convertSwitchToLookupTable(true). 1136 needCanonicalLoops(false). 1137 sinkCommonInsts(true))); 1138 1139 // Optimize parallel scalar instruction chains into SIMD instructions. 1140 if (PTO.SLPVectorization) 1141 OptimizePM.addPass(SLPVectorizerPass()); 1142 1143 // Enhance/cleanup vector code. 1144 OptimizePM.addPass(VectorCombinePass()); 1145 OptimizePM.addPass(InstCombinePass()); 1146 1147 // Unroll small loops to hide loop backedge latency and saturate any parallel 1148 // execution resources of an out-of-order processor. We also then need to 1149 // clean up redundancies and loop invariant code. 1150 // FIXME: It would be really good to use a loop-integrated instruction 1151 // combiner for cleanup here so that the unrolling and LICM can be pipelined 1152 // across the loop nests. 1153 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll 1154 if (EnableUnrollAndJam && PTO.LoopUnrolling) { 1155 OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel())); 1156 } 1157 OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions( 1158 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling, 1159 PTO.ForgetAllSCEVInLoopUnroll))); 1160 OptimizePM.addPass(WarnMissedTransformationsPass()); 1161 OptimizePM.addPass(InstCombinePass()); 1162 OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); 1163 OptimizePM.addPass(createFunctionToLoopPassAdaptor( 1164 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), 1165 EnableMSSALoopDependency, DebugLogging)); 1166 1167 // Now that we've vectorized and unrolled loops, we may have more refined 1168 // alignment information, try to re-derive it here. 1169 OptimizePM.addPass(AlignmentFromAssumptionsPass()); 1170 1171 // Split out cold code. Splitting is done late to avoid hiding context from 1172 // other optimizations and inadvertently regressing performance. The tradeoff 1173 // is that this has a higher code size cost than splitting early. 1174 if (EnableHotColdSplit && !LTOPreLink) 1175 MPM.addPass(HotColdSplittingPass()); 1176 1177 // LoopSink pass sinks instructions hoisted by LICM, which serves as a 1178 // canonicalization pass that enables other optimizations. As a result, 1179 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM 1180 // result too early. 1181 OptimizePM.addPass(LoopSinkPass()); 1182 1183 // And finally clean up LCSSA form before generating code. 1184 OptimizePM.addPass(InstSimplifyPass()); 1185 1186 // This hoists/decomposes div/rem ops. It should run after other sink/hoist 1187 // passes to avoid re-sinking, but before SimplifyCFG because it can allow 1188 // flattening of blocks. 1189 OptimizePM.addPass(DivRemPairsPass()); 1190 1191 // LoopSink (and other loop passes since the last simplifyCFG) might have 1192 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG. 1193 OptimizePM.addPass(SimplifyCFGPass()); 1194 1195 // Optimize PHIs by speculating around them when profitable. Note that this 1196 // pass needs to be run after any PRE or similar pass as it is essentially 1197 // inserting redundancies into the program. This even includes SimplifyCFG. 1198 OptimizePM.addPass(SpeculateAroundPHIsPass()); 1199 1200 if (PTO.Coroutines) 1201 OptimizePM.addPass(CoroCleanupPass()); 1202 1203 // Add the core optimizing pipeline. 1204 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); 1205 1206 for (auto &C : OptimizerLastEPCallbacks) 1207 C(MPM, Level); 1208 1209 if (PTO.CallGraphProfile) 1210 MPM.addPass(CGProfilePass()); 1211 1212 // Now we need to do some global optimization transforms. 1213 // FIXME: It would seem like these should come first in the optimization 1214 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird 1215 // ordering here. 1216 MPM.addPass(GlobalDCEPass()); 1217 MPM.addPass(ConstantMergePass()); 1218 1219 return MPM; 1220 } 1221 1222 ModulePassManager 1223 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, 1224 bool DebugLogging, bool LTOPreLink) { 1225 assert(Level != OptimizationLevel::O0 && 1226 "Must request optimizations for the default pipeline!"); 1227 1228 ModulePassManager MPM(DebugLogging); 1229 1230 // Force any function attributes we want the rest of the pipeline to observe. 1231 MPM.addPass(ForceFunctionAttrsPass()); 1232 1233 // Apply module pipeline start EP callback. 1234 for (auto &C : PipelineStartEPCallbacks) 1235 C(MPM); 1236 1237 if (PGOOpt && PGOOpt->SamplePGOSupport) 1238 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); 1239 1240 // Add the core simplification pipeline. 1241 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None, 1242 DebugLogging)); 1243 1244 // Now add the optimization pipeline. 1245 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink)); 1246 1247 return MPM; 1248 } 1249 1250 ModulePassManager 1251 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level, 1252 bool DebugLogging) { 1253 assert(Level != OptimizationLevel::O0 && 1254 "Must request optimizations for the default pipeline!"); 1255 1256 ModulePassManager MPM(DebugLogging); 1257 1258 // Force any function attributes we want the rest of the pipeline to observe. 1259 MPM.addPass(ForceFunctionAttrsPass()); 1260 1261 if (PGOOpt && PGOOpt->SamplePGOSupport) 1262 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); 1263 1264 // Apply module pipeline start EP callback. 1265 for (auto &C : PipelineStartEPCallbacks) 1266 C(MPM); 1267 1268 // If we are planning to perform ThinLTO later, we don't bloat the code with 1269 // unrolling/vectorization/... now. Just simplify the module as much as we 1270 // can. 1271 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink, 1272 DebugLogging)); 1273 1274 // Run partial inlining pass to partially inline functions that have 1275 // large bodies. 1276 // FIXME: It isn't clear whether this is really the right place to run this 1277 // in ThinLTO. Because there is another canonicalization and simplification 1278 // phase that will run after the thin link, running this here ends up with 1279 // less information than will be available later and it may grow functions in 1280 // ways that aren't beneficial. 1281 if (RunPartialInlining) 1282 MPM.addPass(PartialInlinerPass()); 1283 1284 // Reduce the size of the IR as much as possible. 1285 MPM.addPass(GlobalOptPass()); 1286 1287 // Module simplification splits coroutines, but does not fully clean up 1288 // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up 1289 // on these, we schedule the cleanup here. 1290 if (PTO.Coroutines) 1291 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); 1292 1293 return MPM; 1294 } 1295 1296 ModulePassManager PassBuilder::buildThinLTODefaultPipeline( 1297 OptimizationLevel Level, bool DebugLogging, 1298 const ModuleSummaryIndex *ImportSummary) { 1299 ModulePassManager MPM(DebugLogging); 1300 1301 if (ImportSummary) { 1302 // These passes import type identifier resolutions for whole-program 1303 // devirtualization and CFI. They must run early because other passes may 1304 // disturb the specific instruction patterns that these passes look for, 1305 // creating dependencies on resolutions that may not appear in the summary. 1306 // 1307 // For example, GVN may transform the pattern assume(type.test) appearing in 1308 // two basic blocks into assume(phi(type.test, type.test)), which would 1309 // transform a dependency on a WPD resolution into a dependency on a type 1310 // identifier resolution for CFI. 1311 // 1312 // Also, WPD has access to more precise information than ICP and can 1313 // devirtualize more effectively, so it should operate on the IR first. 1314 // 1315 // The WPD and LowerTypeTest passes need to run at -O0 to lower type 1316 // metadata and intrinsics. 1317 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary)); 1318 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary)); 1319 } 1320 1321 if (Level == OptimizationLevel::O0) 1322 return MPM; 1323 1324 // Force any function attributes we want the rest of the pipeline to observe. 1325 MPM.addPass(ForceFunctionAttrsPass()); 1326 1327 // Add the core simplification pipeline. 1328 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink, 1329 DebugLogging)); 1330 1331 // Now add the optimization pipeline. 1332 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); 1333 1334 return MPM; 1335 } 1336 1337 ModulePassManager 1338 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level, 1339 bool DebugLogging) { 1340 assert(Level != OptimizationLevel::O0 && 1341 "Must request optimizations for the default pipeline!"); 1342 // FIXME: We should use a customized pre-link pipeline! 1343 return buildPerModuleDefaultPipeline(Level, DebugLogging, 1344 /* LTOPreLink */ true); 1345 } 1346 1347 ModulePassManager 1348 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging, 1349 ModuleSummaryIndex *ExportSummary) { 1350 ModulePassManager MPM(DebugLogging); 1351 1352 if (Level == OptimizationLevel::O0) { 1353 // The WPD and LowerTypeTest passes need to run at -O0 to lower type 1354 // metadata and intrinsics. 1355 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); 1356 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); 1357 return MPM; 1358 } 1359 1360 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) { 1361 // Load sample profile before running the LTO optimization pipeline. 1362 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, 1363 PGOOpt->ProfileRemappingFile, 1364 false /* ThinLTOPhase::PreLink */)); 1365 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert 1366 // RequireAnalysisPass for PSI before subsequent non-module passes. 1367 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); 1368 } 1369 1370 // Remove unused virtual tables to improve the quality of code generated by 1371 // whole-program devirtualization and bitset lowering. 1372 MPM.addPass(GlobalDCEPass()); 1373 1374 // Force any function attributes we want the rest of the pipeline to observe. 1375 MPM.addPass(ForceFunctionAttrsPass()); 1376 1377 // Do basic inference of function attributes from known properties of system 1378 // libraries and other oracles. 1379 MPM.addPass(InferFunctionAttrsPass()); 1380 1381 if (Level.getSpeedupLevel() > 1) { 1382 FunctionPassManager EarlyFPM(DebugLogging); 1383 EarlyFPM.addPass(CallSiteSplittingPass()); 1384 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); 1385 1386 // Indirect call promotion. This should promote all the targets that are 1387 // left by the earlier promotion pass that promotes intra-module targets. 1388 // This two-step promotion is to save the compile time. For LTO, it should 1389 // produce the same result as if we only do promotion here. 1390 MPM.addPass(PGOIndirectCallPromotion( 1391 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)); 1392 // Propagate constants at call sites into the functions they call. This 1393 // opens opportunities for globalopt (and inlining) by substituting function 1394 // pointers passed as arguments to direct uses of functions. 1395 MPM.addPass(IPSCCPPass()); 1396 1397 // Attach metadata to indirect call sites indicating the set of functions 1398 // they may target at run-time. This should follow IPSCCP. 1399 MPM.addPass(CalledValuePropagationPass()); 1400 } 1401 1402 // Now deduce any function attributes based in the current code. 1403 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( 1404 PostOrderFunctionAttrsPass())); 1405 1406 // Do RPO function attribute inference across the module to forward-propagate 1407 // attributes where applicable. 1408 // FIXME: Is this really an optimization rather than a canonicalization? 1409 MPM.addPass(ReversePostOrderFunctionAttrsPass()); 1410 1411 // Use in-range annotations on GEP indices to split globals where beneficial. 1412 MPM.addPass(GlobalSplitPass()); 1413 1414 // Run whole program optimization of virtual call when the list of callees 1415 // is fixed. 1416 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); 1417 1418 // Stop here at -O1. 1419 if (Level == OptimizationLevel::O1) { 1420 // The LowerTypeTestsPass needs to run to lower type metadata and the 1421 // type.test intrinsics. The pass does nothing if CFI is disabled. 1422 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); 1423 return MPM; 1424 } 1425 1426 // Optimize globals to try and fold them into constants. 1427 MPM.addPass(GlobalOptPass()); 1428 1429 // Promote any localized globals to SSA registers. 1430 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); 1431 1432 // Linking modules together can lead to duplicate global constant, only 1433 // keep one copy of each constant. 1434 MPM.addPass(ConstantMergePass()); 1435 1436 // Remove unused arguments from functions. 1437 MPM.addPass(DeadArgumentEliminationPass()); 1438 1439 // Reduce the code after globalopt and ipsccp. Both can open up significant 1440 // simplification opportunities, and both can propagate functions through 1441 // function pointers. When this happens, we often have to resolve varargs 1442 // calls, etc, so let instcombine do this. 1443 FunctionPassManager PeepholeFPM(DebugLogging); 1444 if (Level == OptimizationLevel::O3) 1445 PeepholeFPM.addPass(AggressiveInstCombinePass()); 1446 PeepholeFPM.addPass(InstCombinePass()); 1447 invokePeepholeEPCallbacks(PeepholeFPM, Level); 1448 1449 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); 1450 1451 // Note: historically, the PruneEH pass was run first to deduce nounwind and 1452 // generally clean up exception handling overhead. It isn't clear this is 1453 // valuable as the inliner doesn't currently care whether it is inlining an 1454 // invoke or a call. 1455 // Run the inliner now. 1456 MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), 1457 DebugLogging)); 1458 1459 // Optimize globals again after we ran the inliner. 1460 MPM.addPass(GlobalOptPass()); 1461 1462 // Garbage collect dead functions. 1463 // FIXME: Add ArgumentPromotion pass after once it's ported. 1464 MPM.addPass(GlobalDCEPass()); 1465 1466 FunctionPassManager FPM(DebugLogging); 1467 // The IPO Passes may leave cruft around. Clean up after them. 1468 FPM.addPass(InstCombinePass()); 1469 invokePeepholeEPCallbacks(FPM, Level); 1470 1471 FPM.addPass(JumpThreadingPass()); 1472 1473 // Do a post inline PGO instrumentation and use pass. This is a context 1474 // sensitive PGO pass. 1475 if (PGOOpt) { 1476 if (PGOOpt->CSAction == PGOOptions::CSIRInstr) 1477 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true, 1478 /* IsCS */ true, PGOOpt->CSProfileGenFile, 1479 PGOOpt->ProfileRemappingFile); 1480 else if (PGOOpt->CSAction == PGOOptions::CSIRUse) 1481 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false, 1482 /* IsCS */ true, PGOOpt->ProfileFile, 1483 PGOOpt->ProfileRemappingFile); 1484 } 1485 1486 // Break up allocas 1487 FPM.addPass(SROA()); 1488 1489 // LTO provides additional opportunities for tailcall elimination due to 1490 // link-time inlining, and visibility of nocapture attribute. 1491 FPM.addPass(TailCallElimPass()); 1492 1493 // Run a few AA driver optimizations here and now to cleanup the code. 1494 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 1495 1496 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( 1497 PostOrderFunctionAttrsPass())); 1498 // FIXME: here we run IP alias analysis in the legacy PM. 1499 1500 FunctionPassManager MainFPM; 1501 1502 // FIXME: once we fix LoopPass Manager, add LICM here. 1503 // FIXME: once we provide support for enabling MLSM, add it here. 1504 if (RunNewGVN) 1505 MainFPM.addPass(NewGVNPass()); 1506 else 1507 MainFPM.addPass(GVN()); 1508 1509 // Remove dead memcpy()'s. 1510 MainFPM.addPass(MemCpyOptPass()); 1511 1512 // Nuke dead stores. 1513 MainFPM.addPass(DSEPass()); 1514 1515 // FIXME: at this point, we run a bunch of loop passes: 1516 // indVarSimplify, loopDeletion, loopInterchange, loopUnroll, 1517 // loopVectorize. Enable them once the remaining issue with LPM 1518 // are sorted out. 1519 1520 MainFPM.addPass(InstCombinePass()); 1521 MainFPM.addPass(SimplifyCFGPass()); 1522 MainFPM.addPass(SCCPPass()); 1523 MainFPM.addPass(InstCombinePass()); 1524 MainFPM.addPass(BDCEPass()); 1525 1526 // FIXME: We may want to run SLPVectorizer here. 1527 // After vectorization, assume intrinsics may tell us more 1528 // about pointer alignments. 1529 #if 0 1530 MainFPM.add(AlignmentFromAssumptionsPass()); 1531 #endif 1532 1533 // FIXME: Conditionally run LoadCombine here, after it's ported 1534 // (in case we still have this pass, given its questionable usefulness). 1535 1536 MainFPM.addPass(InstCombinePass()); 1537 invokePeepholeEPCallbacks(MainFPM, Level); 1538 MainFPM.addPass(JumpThreadingPass()); 1539 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); 1540 1541 // Create a function that performs CFI checks for cross-DSO calls with 1542 // targets in the current module. 1543 MPM.addPass(CrossDSOCFIPass()); 1544 1545 // Lower type metadata and the type.test intrinsic. This pass supports 1546 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs 1547 // to be run at link time if CFI is enabled. This pass does nothing if 1548 // CFI is disabled. 1549 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); 1550 1551 // Enable splitting late in the FullLTO post-link pipeline. This is done in 1552 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses). 1553 if (EnableHotColdSplit) 1554 MPM.addPass(HotColdSplittingPass()); 1555 1556 // Add late LTO optimization passes. 1557 // Delete basic blocks, which optimization passes may have killed. 1558 MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass())); 1559 1560 // Drop bodies of available eternally objects to improve GlobalDCE. 1561 MPM.addPass(EliminateAvailableExternallyPass()); 1562 1563 // Now that we have optimized the program, discard unreachable functions. 1564 MPM.addPass(GlobalDCEPass()); 1565 1566 // FIXME: Maybe enable MergeFuncs conditionally after it's ported. 1567 return MPM; 1568 } 1569 1570 AAManager PassBuilder::buildDefaultAAPipeline() { 1571 AAManager AA; 1572 1573 // The order in which these are registered determines their priority when 1574 // being queried. 1575 1576 // First we register the basic alias analysis that provides the majority of 1577 // per-function local AA logic. This is a stateless, on-demand local set of 1578 // AA techniques. 1579 AA.registerFunctionAnalysis<BasicAA>(); 1580 1581 // Next we query fast, specialized alias analyses that wrap IR-embedded 1582 // information about aliasing. 1583 AA.registerFunctionAnalysis<ScopedNoAliasAA>(); 1584 AA.registerFunctionAnalysis<TypeBasedAA>(); 1585 1586 // Add support for querying global aliasing information when available. 1587 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module 1588 // analysis, all that the `AAManager` can do is query for any *cached* 1589 // results from `GlobalsAA` through a readonly proxy. 1590 AA.registerModuleAnalysis<GlobalsAA>(); 1591 1592 return AA; 1593 } 1594 1595 static Optional<int> parseRepeatPassName(StringRef Name) { 1596 if (!Name.consume_front("repeat<") || !Name.consume_back(">")) 1597 return None; 1598 int Count; 1599 if (Name.getAsInteger(0, Count) || Count <= 0) 1600 return None; 1601 return Count; 1602 } 1603 1604 static Optional<int> parseDevirtPassName(StringRef Name) { 1605 if (!Name.consume_front("devirt<") || !Name.consume_back(">")) 1606 return None; 1607 int Count; 1608 if (Name.getAsInteger(0, Count) || Count <= 0) 1609 return None; 1610 return Count; 1611 } 1612 1613 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) { 1614 if (!Name.consume_front(PassName)) 1615 return false; 1616 // normal pass name w/o parameters == default parameters 1617 if (Name.empty()) 1618 return true; 1619 return Name.startswith("<") && Name.endswith(">"); 1620 } 1621 1622 namespace { 1623 1624 /// This performs customized parsing of pass name with parameters. 1625 /// 1626 /// We do not need parametrization of passes in textual pipeline very often, 1627 /// yet on a rare occasion ability to specify parameters right there can be 1628 /// useful. 1629 /// 1630 /// \p Name - parameterized specification of a pass from a textual pipeline 1631 /// is a string in a form of : 1632 /// PassName '<' parameter-list '>' 1633 /// 1634 /// Parameter list is being parsed by the parser callable argument, \p Parser, 1635 /// It takes a string-ref of parameters and returns either StringError or a 1636 /// parameter list in a form of a custom parameters type, all wrapped into 1637 /// Expected<> template class. 1638 /// 1639 template <typename ParametersParseCallableT> 1640 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name, 1641 StringRef PassName) -> decltype(Parser(StringRef{})) { 1642 using ParametersT = typename decltype(Parser(StringRef{}))::value_type; 1643 1644 StringRef Params = Name; 1645 if (!Params.consume_front(PassName)) { 1646 assert(false && 1647 "unable to strip pass name from parametrized pass specification"); 1648 } 1649 if (Params.empty()) 1650 return ParametersT{}; 1651 if (!Params.consume_front("<") || !Params.consume_back(">")) { 1652 assert(false && "invalid format for parametrized pass name"); 1653 } 1654 1655 Expected<ParametersT> Result = Parser(Params); 1656 assert((Result || Result.template errorIsA<StringError>()) && 1657 "Pass parameter parser can only return StringErrors."); 1658 return Result; 1659 } 1660 1661 /// Parser of parameters for LoopUnroll pass. 1662 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) { 1663 LoopUnrollOptions UnrollOpts; 1664 while (!Params.empty()) { 1665 StringRef ParamName; 1666 std::tie(ParamName, Params) = Params.split(';'); 1667 int OptLevel = StringSwitch<int>(ParamName) 1668 .Case("O0", 0) 1669 .Case("O1", 1) 1670 .Case("O2", 2) 1671 .Case("O3", 3) 1672 .Default(-1); 1673 if (OptLevel >= 0) { 1674 UnrollOpts.setOptLevel(OptLevel); 1675 continue; 1676 } 1677 if (ParamName.consume_front("full-unroll-max=")) { 1678 int Count; 1679 if (ParamName.getAsInteger(0, Count)) 1680 return make_error<StringError>( 1681 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(), 1682 inconvertibleErrorCode()); 1683 UnrollOpts.setFullUnrollMaxCount(Count); 1684 continue; 1685 } 1686 1687 bool Enable = !ParamName.consume_front("no-"); 1688 if (ParamName == "partial") { 1689 UnrollOpts.setPartial(Enable); 1690 } else if (ParamName == "peeling") { 1691 UnrollOpts.setPeeling(Enable); 1692 } else if (ParamName == "profile-peeling") { 1693 UnrollOpts.setProfileBasedPeeling(Enable); 1694 } else if (ParamName == "runtime") { 1695 UnrollOpts.setRuntime(Enable); 1696 } else if (ParamName == "upperbound") { 1697 UnrollOpts.setUpperBound(Enable); 1698 } else { 1699 return make_error<StringError>( 1700 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(), 1701 inconvertibleErrorCode()); 1702 } 1703 } 1704 return UnrollOpts; 1705 } 1706 1707 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) { 1708 MemorySanitizerOptions Result; 1709 while (!Params.empty()) { 1710 StringRef ParamName; 1711 std::tie(ParamName, Params) = Params.split(';'); 1712 1713 if (ParamName == "recover") { 1714 Result.Recover = true; 1715 } else if (ParamName == "kernel") { 1716 Result.Kernel = true; 1717 } else if (ParamName.consume_front("track-origins=")) { 1718 if (ParamName.getAsInteger(0, Result.TrackOrigins)) 1719 return make_error<StringError>( 1720 formatv("invalid argument to MemorySanitizer pass track-origins " 1721 "parameter: '{0}' ", 1722 ParamName) 1723 .str(), 1724 inconvertibleErrorCode()); 1725 } else { 1726 return make_error<StringError>( 1727 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName) 1728 .str(), 1729 inconvertibleErrorCode()); 1730 } 1731 } 1732 return Result; 1733 } 1734 1735 /// Parser of parameters for SimplifyCFG pass. 1736 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) { 1737 SimplifyCFGOptions Result; 1738 while (!Params.empty()) { 1739 StringRef ParamName; 1740 std::tie(ParamName, Params) = Params.split(';'); 1741 1742 bool Enable = !ParamName.consume_front("no-"); 1743 if (ParamName == "forward-switch-cond") { 1744 Result.forwardSwitchCondToPhi(Enable); 1745 } else if (ParamName == "switch-to-lookup") { 1746 Result.convertSwitchToLookupTable(Enable); 1747 } else if (ParamName == "keep-loops") { 1748 Result.needCanonicalLoops(Enable); 1749 } else if (ParamName == "sink-common-insts") { 1750 Result.sinkCommonInsts(Enable); 1751 } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) { 1752 APInt BonusInstThreshold; 1753 if (ParamName.getAsInteger(0, BonusInstThreshold)) 1754 return make_error<StringError>( 1755 formatv("invalid argument to SimplifyCFG pass bonus-threshold " 1756 "parameter: '{0}' ", 1757 ParamName).str(), 1758 inconvertibleErrorCode()); 1759 Result.bonusInstThreshold(BonusInstThreshold.getSExtValue()); 1760 } else { 1761 return make_error<StringError>( 1762 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(), 1763 inconvertibleErrorCode()); 1764 } 1765 } 1766 return Result; 1767 } 1768 1769 /// Parser of parameters for LoopVectorize pass. 1770 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) { 1771 LoopVectorizeOptions Opts; 1772 while (!Params.empty()) { 1773 StringRef ParamName; 1774 std::tie(ParamName, Params) = Params.split(';'); 1775 1776 bool Enable = !ParamName.consume_front("no-"); 1777 if (ParamName == "interleave-forced-only") { 1778 Opts.setInterleaveOnlyWhenForced(Enable); 1779 } else if (ParamName == "vectorize-forced-only") { 1780 Opts.setVectorizeOnlyWhenForced(Enable); 1781 } else { 1782 return make_error<StringError>( 1783 formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(), 1784 inconvertibleErrorCode()); 1785 } 1786 } 1787 return Opts; 1788 } 1789 1790 Expected<bool> parseLoopUnswitchOptions(StringRef Params) { 1791 bool Result = false; 1792 while (!Params.empty()) { 1793 StringRef ParamName; 1794 std::tie(ParamName, Params) = Params.split(';'); 1795 1796 bool Enable = !ParamName.consume_front("no-"); 1797 if (ParamName == "nontrivial") { 1798 Result = Enable; 1799 } else { 1800 return make_error<StringError>( 1801 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName) 1802 .str(), 1803 inconvertibleErrorCode()); 1804 } 1805 } 1806 return Result; 1807 } 1808 1809 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) { 1810 bool Result = false; 1811 while (!Params.empty()) { 1812 StringRef ParamName; 1813 std::tie(ParamName, Params) = Params.split(';'); 1814 1815 bool Enable = !ParamName.consume_front("no-"); 1816 if (ParamName == "split-footer-bb") { 1817 Result = Enable; 1818 } else { 1819 return make_error<StringError>( 1820 formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ", 1821 ParamName) 1822 .str(), 1823 inconvertibleErrorCode()); 1824 } 1825 } 1826 return Result; 1827 } 1828 1829 Expected<GVNOptions> parseGVNOptions(StringRef Params) { 1830 GVNOptions Result; 1831 while (!Params.empty()) { 1832 StringRef ParamName; 1833 std::tie(ParamName, Params) = Params.split(';'); 1834 1835 bool Enable = !ParamName.consume_front("no-"); 1836 if (ParamName == "pre") { 1837 Result.setPRE(Enable); 1838 } else if (ParamName == "load-pre") { 1839 Result.setLoadPRE(Enable); 1840 } else if (ParamName == "memdep") { 1841 Result.setMemDep(Enable); 1842 } else { 1843 return make_error<StringError>( 1844 formatv("invalid GVN pass parameter '{0}' ", ParamName).str(), 1845 inconvertibleErrorCode()); 1846 } 1847 } 1848 return Result; 1849 } 1850 1851 Expected<StackLifetime::LivenessType> 1852 parseStackLifetimeOptions(StringRef Params) { 1853 StackLifetime::LivenessType Result = StackLifetime::LivenessType::May; 1854 while (!Params.empty()) { 1855 StringRef ParamName; 1856 std::tie(ParamName, Params) = Params.split(';'); 1857 1858 if (ParamName == "may") { 1859 Result = StackLifetime::LivenessType::May; 1860 } else if (ParamName == "must") { 1861 Result = StackLifetime::LivenessType::Must; 1862 } else { 1863 return make_error<StringError>( 1864 formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(), 1865 inconvertibleErrorCode()); 1866 } 1867 } 1868 return Result; 1869 } 1870 1871 } // namespace 1872 1873 /// Tests whether a pass name starts with a valid prefix for a default pipeline 1874 /// alias. 1875 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { 1876 return Name.startswith("default") || Name.startswith("thinlto") || 1877 Name.startswith("lto"); 1878 } 1879 1880 /// Tests whether registered callbacks will accept a given pass name. 1881 /// 1882 /// When parsing a pipeline text, the type of the outermost pipeline may be 1883 /// omitted, in which case the type is automatically determined from the first 1884 /// pass name in the text. This may be a name that is handled through one of the 1885 /// callbacks. We check this through the oridinary parsing callbacks by setting 1886 /// up a dummy PassManager in order to not force the client to also handle this 1887 /// type of query. 1888 template <typename PassManagerT, typename CallbacksT> 1889 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) { 1890 if (!Callbacks.empty()) { 1891 PassManagerT DummyPM; 1892 for (auto &CB : Callbacks) 1893 if (CB(Name, DummyPM, {})) 1894 return true; 1895 } 1896 return false; 1897 } 1898 1899 template <typename CallbacksT> 1900 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { 1901 // Manually handle aliases for pre-configured pipeline fragments. 1902 if (startsWithDefaultPipelineAliasPrefix(Name)) 1903 return DefaultAliasRegex.match(Name); 1904 1905 // Explicitly handle pass manager names. 1906 if (Name == "module") 1907 return true; 1908 if (Name == "cgscc") 1909 return true; 1910 if (Name == "function") 1911 return true; 1912 1913 // Explicitly handle custom-parsed pass names. 1914 if (parseRepeatPassName(Name)) 1915 return true; 1916 1917 #define MODULE_PASS(NAME, CREATE_PASS) \ 1918 if (Name == NAME) \ 1919 return true; 1920 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 1921 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1922 return true; 1923 #include "PassRegistry.def" 1924 1925 return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks); 1926 } 1927 1928 template <typename CallbacksT> 1929 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { 1930 // Explicitly handle pass manager names. 1931 if (Name == "cgscc") 1932 return true; 1933 if (Name == "function") 1934 return true; 1935 1936 // Explicitly handle custom-parsed pass names. 1937 if (parseRepeatPassName(Name)) 1938 return true; 1939 if (parseDevirtPassName(Name)) 1940 return true; 1941 1942 #define CGSCC_PASS(NAME, CREATE_PASS) \ 1943 if (Name == NAME) \ 1944 return true; 1945 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 1946 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1947 return true; 1948 #include "PassRegistry.def" 1949 1950 return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks); 1951 } 1952 1953 template <typename CallbacksT> 1954 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { 1955 // Explicitly handle pass manager names. 1956 if (Name == "function") 1957 return true; 1958 if (Name == "loop" || Name == "loop-mssa") 1959 return true; 1960 1961 // Explicitly handle custom-parsed pass names. 1962 if (parseRepeatPassName(Name)) 1963 return true; 1964 1965 #define FUNCTION_PASS(NAME, CREATE_PASS) \ 1966 if (Name == NAME) \ 1967 return true; 1968 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ 1969 if (checkParametrizedPassName(Name, NAME)) \ 1970 return true; 1971 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 1972 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1973 return true; 1974 #include "PassRegistry.def" 1975 1976 return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks); 1977 } 1978 1979 template <typename CallbacksT> 1980 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) { 1981 // Explicitly handle pass manager names. 1982 if (Name == "loop" || Name == "loop-mssa") 1983 return true; 1984 1985 // Explicitly handle custom-parsed pass names. 1986 if (parseRepeatPassName(Name)) 1987 return true; 1988 1989 #define LOOP_PASS(NAME, CREATE_PASS) \ 1990 if (Name == NAME) \ 1991 return true; 1992 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ 1993 if (checkParametrizedPassName(Name, NAME)) \ 1994 return true; 1995 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 1996 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1997 return true; 1998 #include "PassRegistry.def" 1999 2000 return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks); 2001 } 2002 2003 Optional<std::vector<PassBuilder::PipelineElement>> 2004 PassBuilder::parsePipelineText(StringRef Text) { 2005 std::vector<PipelineElement> ResultPipeline; 2006 2007 SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = { 2008 &ResultPipeline}; 2009 for (;;) { 2010 std::vector<PipelineElement> &Pipeline = *PipelineStack.back(); 2011 size_t Pos = Text.find_first_of(",()"); 2012 Pipeline.push_back({Text.substr(0, Pos), {}}); 2013 2014 // If we have a single terminating name, we're done. 2015 if (Pos == Text.npos) 2016 break; 2017 2018 char Sep = Text[Pos]; 2019 Text = Text.substr(Pos + 1); 2020 if (Sep == ',') 2021 // Just a name ending in a comma, continue. 2022 continue; 2023 2024 if (Sep == '(') { 2025 // Push the inner pipeline onto the stack to continue processing. 2026 PipelineStack.push_back(&Pipeline.back().InnerPipeline); 2027 continue; 2028 } 2029 2030 assert(Sep == ')' && "Bogus separator!"); 2031 // When handling the close parenthesis, we greedily consume them to avoid 2032 // empty strings in the pipeline. 2033 do { 2034 // If we try to pop the outer pipeline we have unbalanced parentheses. 2035 if (PipelineStack.size() == 1) 2036 return None; 2037 2038 PipelineStack.pop_back(); 2039 } while (Text.consume_front(")")); 2040 2041 // Check if we've finished parsing. 2042 if (Text.empty()) 2043 break; 2044 2045 // Otherwise, the end of an inner pipeline always has to be followed by 2046 // a comma, and then we can continue. 2047 if (!Text.consume_front(",")) 2048 return None; 2049 } 2050 2051 if (PipelineStack.size() > 1) 2052 // Unbalanced paretheses. 2053 return None; 2054 2055 assert(PipelineStack.back() == &ResultPipeline && 2056 "Wrong pipeline at the bottom of the stack!"); 2057 return {std::move(ResultPipeline)}; 2058 } 2059 2060 Error PassBuilder::parseModulePass(ModulePassManager &MPM, 2061 const PipelineElement &E, 2062 bool VerifyEachPass, bool DebugLogging) { 2063 auto &Name = E.Name; 2064 auto &InnerPipeline = E.InnerPipeline; 2065 2066 // First handle complex passes like the pass managers which carry pipelines. 2067 if (!InnerPipeline.empty()) { 2068 if (Name == "module") { 2069 ModulePassManager NestedMPM(DebugLogging); 2070 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, 2071 VerifyEachPass, DebugLogging)) 2072 return Err; 2073 MPM.addPass(std::move(NestedMPM)); 2074 return Error::success(); 2075 } 2076 if (Name == "cgscc") { 2077 CGSCCPassManager CGPM(DebugLogging); 2078 if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass, 2079 DebugLogging)) 2080 return Err; 2081 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); 2082 return Error::success(); 2083 } 2084 if (Name == "function") { 2085 FunctionPassManager FPM(DebugLogging); 2086 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, 2087 VerifyEachPass, DebugLogging)) 2088 return Err; 2089 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 2090 return Error::success(); 2091 } 2092 if (auto Count = parseRepeatPassName(Name)) { 2093 ModulePassManager NestedMPM(DebugLogging); 2094 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, 2095 VerifyEachPass, DebugLogging)) 2096 return Err; 2097 MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM))); 2098 return Error::success(); 2099 } 2100 2101 for (auto &C : ModulePipelineParsingCallbacks) 2102 if (C(Name, MPM, InnerPipeline)) 2103 return Error::success(); 2104 2105 // Normal passes can't have pipelines. 2106 return make_error<StringError>( 2107 formatv("invalid use of '{0}' pass as module pipeline", Name).str(), 2108 inconvertibleErrorCode()); 2109 ; 2110 } 2111 2112 // Manually handle aliases for pre-configured pipeline fragments. 2113 if (startsWithDefaultPipelineAliasPrefix(Name)) { 2114 SmallVector<StringRef, 3> Matches; 2115 if (!DefaultAliasRegex.match(Name, &Matches)) 2116 return make_error<StringError>( 2117 formatv("unknown default pipeline alias '{0}'", Name).str(), 2118 inconvertibleErrorCode()); 2119 2120 assert(Matches.size() == 3 && "Must capture two matched strings!"); 2121 2122 OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2]) 2123 .Case("O0", OptimizationLevel::O0) 2124 .Case("O1", OptimizationLevel::O1) 2125 .Case("O2", OptimizationLevel::O2) 2126 .Case("O3", OptimizationLevel::O3) 2127 .Case("Os", OptimizationLevel::Os) 2128 .Case("Oz", OptimizationLevel::Oz); 2129 if (L == OptimizationLevel::O0) { 2130 // Add instrumentation PGO passes -- at O0 we can still do PGO. 2131 if (PGOOpt && Matches[1] != "thinlto" && 2132 (PGOOpt->Action == PGOOptions::IRInstr || 2133 PGOOpt->Action == PGOOptions::IRUse)) 2134 addPGOInstrPassesForO0( 2135 MPM, DebugLogging, 2136 /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr), 2137 /* IsCS */ false, PGOOpt->ProfileFile, 2138 PGOOpt->ProfileRemappingFile); 2139 2140 // For IR that makes use of coroutines intrinsics, coroutine passes must 2141 // be run, even at -O0. 2142 if (PTO.Coroutines) { 2143 MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass())); 2144 2145 CGSCCPassManager CGPM(DebugLogging); 2146 CGPM.addPass(CoroSplitPass()); 2147 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); 2148 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); 2149 2150 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); 2151 } 2152 2153 // Do nothing else at all! 2154 return Error::success(); 2155 } 2156 2157 // This is consistent with old pass manager invoked via opt, but 2158 // inconsistent with clang. Clang doesn't enable loop vectorization 2159 // but does enable slp vectorization at Oz. 2160 PTO.LoopVectorization = 2161 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz; 2162 PTO.SLPVectorization = 2163 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz; 2164 2165 if (Matches[1] == "default") { 2166 MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging)); 2167 } else if (Matches[1] == "thinlto-pre-link") { 2168 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging)); 2169 } else if (Matches[1] == "thinlto") { 2170 MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr)); 2171 } else if (Matches[1] == "lto-pre-link") { 2172 MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging)); 2173 } else { 2174 assert(Matches[1] == "lto" && "Not one of the matched options!"); 2175 MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr)); 2176 } 2177 return Error::success(); 2178 } 2179 2180 // Finally expand the basic registered passes from the .inc file. 2181 #define MODULE_PASS(NAME, CREATE_PASS) \ 2182 if (Name == NAME) { \ 2183 MPM.addPass(CREATE_PASS); \ 2184 return Error::success(); \ 2185 } 2186 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 2187 if (Name == "require<" NAME ">") { \ 2188 MPM.addPass( \ 2189 RequireAnalysisPass< \ 2190 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \ 2191 return Error::success(); \ 2192 } \ 2193 if (Name == "invalidate<" NAME ">") { \ 2194 MPM.addPass(InvalidateAnalysisPass< \ 2195 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 2196 return Error::success(); \ 2197 } 2198 #include "PassRegistry.def" 2199 2200 for (auto &C : ModulePipelineParsingCallbacks) 2201 if (C(Name, MPM, InnerPipeline)) 2202 return Error::success(); 2203 return make_error<StringError>( 2204 formatv("unknown module pass '{0}'", Name).str(), 2205 inconvertibleErrorCode()); 2206 } 2207 2208 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, 2209 const PipelineElement &E, bool VerifyEachPass, 2210 bool DebugLogging) { 2211 auto &Name = E.Name; 2212 auto &InnerPipeline = E.InnerPipeline; 2213 2214 // First handle complex passes like the pass managers which carry pipelines. 2215 if (!InnerPipeline.empty()) { 2216 if (Name == "cgscc") { 2217 CGSCCPassManager NestedCGPM(DebugLogging); 2218 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, 2219 VerifyEachPass, DebugLogging)) 2220 return Err; 2221 // Add the nested pass manager with the appropriate adaptor. 2222 CGPM.addPass(std::move(NestedCGPM)); 2223 return Error::success(); 2224 } 2225 if (Name == "function") { 2226 FunctionPassManager FPM(DebugLogging); 2227 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, 2228 VerifyEachPass, DebugLogging)) 2229 return Err; 2230 // Add the nested pass manager with the appropriate adaptor. 2231 CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); 2232 return Error::success(); 2233 } 2234 if (auto Count = parseRepeatPassName(Name)) { 2235 CGSCCPassManager NestedCGPM(DebugLogging); 2236 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, 2237 VerifyEachPass, DebugLogging)) 2238 return Err; 2239 CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM))); 2240 return Error::success(); 2241 } 2242 if (auto MaxRepetitions = parseDevirtPassName(Name)) { 2243 CGSCCPassManager NestedCGPM(DebugLogging); 2244 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, 2245 VerifyEachPass, DebugLogging)) 2246 return Err; 2247 CGPM.addPass( 2248 createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions)); 2249 return Error::success(); 2250 } 2251 2252 for (auto &C : CGSCCPipelineParsingCallbacks) 2253 if (C(Name, CGPM, InnerPipeline)) 2254 return Error::success(); 2255 2256 // Normal passes can't have pipelines. 2257 return make_error<StringError>( 2258 formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(), 2259 inconvertibleErrorCode()); 2260 } 2261 2262 // Now expand the basic registered passes from the .inc file. 2263 #define CGSCC_PASS(NAME, CREATE_PASS) \ 2264 if (Name == NAME) { \ 2265 CGPM.addPass(CREATE_PASS); \ 2266 return Error::success(); \ 2267 } 2268 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 2269 if (Name == "require<" NAME ">") { \ 2270 CGPM.addPass(RequireAnalysisPass< \ 2271 std::remove_reference<decltype(CREATE_PASS)>::type, \ 2272 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \ 2273 CGSCCUpdateResult &>()); \ 2274 return Error::success(); \ 2275 } \ 2276 if (Name == "invalidate<" NAME ">") { \ 2277 CGPM.addPass(InvalidateAnalysisPass< \ 2278 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 2279 return Error::success(); \ 2280 } 2281 #include "PassRegistry.def" 2282 2283 for (auto &C : CGSCCPipelineParsingCallbacks) 2284 if (C(Name, CGPM, InnerPipeline)) 2285 return Error::success(); 2286 return make_error<StringError>( 2287 formatv("unknown cgscc pass '{0}'", Name).str(), 2288 inconvertibleErrorCode()); 2289 } 2290 2291 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM, 2292 const PipelineElement &E, 2293 bool VerifyEachPass, bool DebugLogging) { 2294 auto &Name = E.Name; 2295 auto &InnerPipeline = E.InnerPipeline; 2296 2297 // First handle complex passes like the pass managers which carry pipelines. 2298 if (!InnerPipeline.empty()) { 2299 if (Name == "function") { 2300 FunctionPassManager NestedFPM(DebugLogging); 2301 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, 2302 VerifyEachPass, DebugLogging)) 2303 return Err; 2304 // Add the nested pass manager with the appropriate adaptor. 2305 FPM.addPass(std::move(NestedFPM)); 2306 return Error::success(); 2307 } 2308 if (Name == "loop" || Name == "loop-mssa") { 2309 LoopPassManager LPM(DebugLogging); 2310 if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass, 2311 DebugLogging)) 2312 return Err; 2313 // Add the nested pass manager with the appropriate adaptor. 2314 bool UseMemorySSA = (Name == "loop-mssa"); 2315 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA, 2316 DebugLogging)); 2317 return Error::success(); 2318 } 2319 if (auto Count = parseRepeatPassName(Name)) { 2320 FunctionPassManager NestedFPM(DebugLogging); 2321 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, 2322 VerifyEachPass, DebugLogging)) 2323 return Err; 2324 FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM))); 2325 return Error::success(); 2326 } 2327 2328 for (auto &C : FunctionPipelineParsingCallbacks) 2329 if (C(Name, FPM, InnerPipeline)) 2330 return Error::success(); 2331 2332 // Normal passes can't have pipelines. 2333 return make_error<StringError>( 2334 formatv("invalid use of '{0}' pass as function pipeline", Name).str(), 2335 inconvertibleErrorCode()); 2336 } 2337 2338 // Now expand the basic registered passes from the .inc file. 2339 #define FUNCTION_PASS(NAME, CREATE_PASS) \ 2340 if (Name == NAME) { \ 2341 FPM.addPass(CREATE_PASS); \ 2342 return Error::success(); \ 2343 } 2344 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ 2345 if (checkParametrizedPassName(Name, NAME)) { \ 2346 auto Params = parsePassParameters(PARSER, Name, NAME); \ 2347 if (!Params) \ 2348 return Params.takeError(); \ 2349 FPM.addPass(CREATE_PASS(Params.get())); \ 2350 return Error::success(); \ 2351 } 2352 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 2353 if (Name == "require<" NAME ">") { \ 2354 FPM.addPass( \ 2355 RequireAnalysisPass< \ 2356 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \ 2357 return Error::success(); \ 2358 } \ 2359 if (Name == "invalidate<" NAME ">") { \ 2360 FPM.addPass(InvalidateAnalysisPass< \ 2361 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 2362 return Error::success(); \ 2363 } 2364 #include "PassRegistry.def" 2365 2366 for (auto &C : FunctionPipelineParsingCallbacks) 2367 if (C(Name, FPM, InnerPipeline)) 2368 return Error::success(); 2369 return make_error<StringError>( 2370 formatv("unknown function pass '{0}'", Name).str(), 2371 inconvertibleErrorCode()); 2372 } 2373 2374 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, 2375 bool VerifyEachPass, bool DebugLogging) { 2376 StringRef Name = E.Name; 2377 auto &InnerPipeline = E.InnerPipeline; 2378 2379 // First handle complex passes like the pass managers which carry pipelines. 2380 if (!InnerPipeline.empty()) { 2381 if (Name == "loop") { 2382 LoopPassManager NestedLPM(DebugLogging); 2383 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, 2384 VerifyEachPass, DebugLogging)) 2385 return Err; 2386 // Add the nested pass manager with the appropriate adaptor. 2387 LPM.addPass(std::move(NestedLPM)); 2388 return Error::success(); 2389 } 2390 if (auto Count = parseRepeatPassName(Name)) { 2391 LoopPassManager NestedLPM(DebugLogging); 2392 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, 2393 VerifyEachPass, DebugLogging)) 2394 return Err; 2395 LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM))); 2396 return Error::success(); 2397 } 2398 2399 for (auto &C : LoopPipelineParsingCallbacks) 2400 if (C(Name, LPM, InnerPipeline)) 2401 return Error::success(); 2402 2403 // Normal passes can't have pipelines. 2404 return make_error<StringError>( 2405 formatv("invalid use of '{0}' pass as loop pipeline", Name).str(), 2406 inconvertibleErrorCode()); 2407 } 2408 2409 // Now expand the basic registered passes from the .inc file. 2410 #define LOOP_PASS(NAME, CREATE_PASS) \ 2411 if (Name == NAME) { \ 2412 LPM.addPass(CREATE_PASS); \ 2413 return Error::success(); \ 2414 } 2415 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ 2416 if (checkParametrizedPassName(Name, NAME)) { \ 2417 auto Params = parsePassParameters(PARSER, Name, NAME); \ 2418 if (!Params) \ 2419 return Params.takeError(); \ 2420 LPM.addPass(CREATE_PASS(Params.get())); \ 2421 return Error::success(); \ 2422 } 2423 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 2424 if (Name == "require<" NAME ">") { \ 2425 LPM.addPass(RequireAnalysisPass< \ 2426 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \ 2427 LoopAnalysisManager, LoopStandardAnalysisResults &, \ 2428 LPMUpdater &>()); \ 2429 return Error::success(); \ 2430 } \ 2431 if (Name == "invalidate<" NAME ">") { \ 2432 LPM.addPass(InvalidateAnalysisPass< \ 2433 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 2434 return Error::success(); \ 2435 } 2436 #include "PassRegistry.def" 2437 2438 for (auto &C : LoopPipelineParsingCallbacks) 2439 if (C(Name, LPM, InnerPipeline)) 2440 return Error::success(); 2441 return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(), 2442 inconvertibleErrorCode()); 2443 } 2444 2445 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) { 2446 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ 2447 if (Name == NAME) { \ 2448 AA.registerModuleAnalysis< \ 2449 std::remove_reference<decltype(CREATE_PASS)>::type>(); \ 2450 return true; \ 2451 } 2452 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ 2453 if (Name == NAME) { \ 2454 AA.registerFunctionAnalysis< \ 2455 std::remove_reference<decltype(CREATE_PASS)>::type>(); \ 2456 return true; \ 2457 } 2458 #include "PassRegistry.def" 2459 2460 for (auto &C : AAParsingCallbacks) 2461 if (C(Name, AA)) 2462 return true; 2463 return false; 2464 } 2465 2466 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM, 2467 ArrayRef<PipelineElement> Pipeline, 2468 bool VerifyEachPass, 2469 bool DebugLogging) { 2470 for (const auto &Element : Pipeline) { 2471 if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging)) 2472 return Err; 2473 // FIXME: No verifier support for Loop passes! 2474 } 2475 return Error::success(); 2476 } 2477 2478 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM, 2479 ArrayRef<PipelineElement> Pipeline, 2480 bool VerifyEachPass, 2481 bool DebugLogging) { 2482 for (const auto &Element : Pipeline) { 2483 if (auto Err = 2484 parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging)) 2485 return Err; 2486 if (VerifyEachPass) 2487 FPM.addPass(VerifierPass()); 2488 } 2489 return Error::success(); 2490 } 2491 2492 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM, 2493 ArrayRef<PipelineElement> Pipeline, 2494 bool VerifyEachPass, 2495 bool DebugLogging) { 2496 for (const auto &Element : Pipeline) { 2497 if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging)) 2498 return Err; 2499 // FIXME: No verifier support for CGSCC passes! 2500 } 2501 return Error::success(); 2502 } 2503 2504 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM, 2505 FunctionAnalysisManager &FAM, 2506 CGSCCAnalysisManager &CGAM, 2507 ModuleAnalysisManager &MAM) { 2508 MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); }); 2509 MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); }); 2510 CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); }); 2511 FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); }); 2512 FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); }); 2513 FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); }); 2514 LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); }); 2515 } 2516 2517 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, 2518 ArrayRef<PipelineElement> Pipeline, 2519 bool VerifyEachPass, 2520 bool DebugLogging) { 2521 for (const auto &Element : Pipeline) { 2522 if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging)) 2523 return Err; 2524 if (VerifyEachPass) 2525 MPM.addPass(VerifierPass()); 2526 } 2527 return Error::success(); 2528 } 2529 2530 // Primary pass pipeline description parsing routine for a \c ModulePassManager 2531 // FIXME: Should this routine accept a TargetMachine or require the caller to 2532 // pre-populate the analysis managers with target-specific stuff? 2533 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM, 2534 StringRef PipelineText, 2535 bool VerifyEachPass, bool DebugLogging) { 2536 auto Pipeline = parsePipelineText(PipelineText); 2537 if (!Pipeline || Pipeline->empty()) 2538 return make_error<StringError>( 2539 formatv("invalid pipeline '{0}'", PipelineText).str(), 2540 inconvertibleErrorCode()); 2541 2542 // If the first name isn't at the module layer, wrap the pipeline up 2543 // automatically. 2544 StringRef FirstName = Pipeline->front().Name; 2545 2546 if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) { 2547 if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) { 2548 Pipeline = {{"cgscc", std::move(*Pipeline)}}; 2549 } else if (isFunctionPassName(FirstName, 2550 FunctionPipelineParsingCallbacks)) { 2551 Pipeline = {{"function", std::move(*Pipeline)}}; 2552 } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) { 2553 Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}}; 2554 } else { 2555 for (auto &C : TopLevelPipelineParsingCallbacks) 2556 if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging)) 2557 return Error::success(); 2558 2559 // Unknown pass or pipeline name! 2560 auto &InnerPipeline = Pipeline->front().InnerPipeline; 2561 return make_error<StringError>( 2562 formatv("unknown {0} name '{1}'", 2563 (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName) 2564 .str(), 2565 inconvertibleErrorCode()); 2566 } 2567 } 2568 2569 if (auto Err = 2570 parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging)) 2571 return Err; 2572 return Error::success(); 2573 } 2574 2575 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager 2576 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM, 2577 StringRef PipelineText, 2578 bool VerifyEachPass, bool DebugLogging) { 2579 auto Pipeline = parsePipelineText(PipelineText); 2580 if (!Pipeline || Pipeline->empty()) 2581 return make_error<StringError>( 2582 formatv("invalid pipeline '{0}'", PipelineText).str(), 2583 inconvertibleErrorCode()); 2584 2585 StringRef FirstName = Pipeline->front().Name; 2586 if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) 2587 return make_error<StringError>( 2588 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName, 2589 PipelineText) 2590 .str(), 2591 inconvertibleErrorCode()); 2592 2593 if (auto Err = 2594 parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) 2595 return Err; 2596 return Error::success(); 2597 } 2598 2599 // Primary pass pipeline description parsing routine for a \c 2600 // FunctionPassManager 2601 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM, 2602 StringRef PipelineText, 2603 bool VerifyEachPass, bool DebugLogging) { 2604 auto Pipeline = parsePipelineText(PipelineText); 2605 if (!Pipeline || Pipeline->empty()) 2606 return make_error<StringError>( 2607 formatv("invalid pipeline '{0}'", PipelineText).str(), 2608 inconvertibleErrorCode()); 2609 2610 StringRef FirstName = Pipeline->front().Name; 2611 if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) 2612 return make_error<StringError>( 2613 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName, 2614 PipelineText) 2615 .str(), 2616 inconvertibleErrorCode()); 2617 2618 if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass, 2619 DebugLogging)) 2620 return Err; 2621 return Error::success(); 2622 } 2623 2624 // Primary pass pipeline description parsing routine for a \c LoopPassManager 2625 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM, 2626 StringRef PipelineText, 2627 bool VerifyEachPass, bool DebugLogging) { 2628 auto Pipeline = parsePipelineText(PipelineText); 2629 if (!Pipeline || Pipeline->empty()) 2630 return make_error<StringError>( 2631 formatv("invalid pipeline '{0}'", PipelineText).str(), 2632 inconvertibleErrorCode()); 2633 2634 if (auto Err = 2635 parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) 2636 return Err; 2637 2638 return Error::success(); 2639 } 2640 2641 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { 2642 // If the pipeline just consists of the word 'default' just replace the AA 2643 // manager with our default one. 2644 if (PipelineText == "default") { 2645 AA = buildDefaultAAPipeline(); 2646 return Error::success(); 2647 } 2648 2649 while (!PipelineText.empty()) { 2650 StringRef Name; 2651 std::tie(Name, PipelineText) = PipelineText.split(','); 2652 if (!parseAAPassName(AA, Name)) 2653 return make_error<StringError>( 2654 formatv("unknown alias analysis name '{0}'", Name).str(), 2655 inconvertibleErrorCode()); 2656 } 2657 2658 return Error::success(); 2659 } 2660 2661 bool PassBuilder::isAAPassName(StringRef PassName) { 2662 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ 2663 if (PassName == NAME) \ 2664 return true; 2665 #include "PassRegistry.def" 2666 return false; 2667 } 2668 2669 bool PassBuilder::isAnalysisPassName(StringRef PassName) { 2670 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 2671 if (PassName == NAME) \ 2672 return true; 2673 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 2674 if (PassName == NAME) \ 2675 return true; 2676 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 2677 if (PassName == NAME) \ 2678 return true; 2679 #define CGSSC_ANALYSIS(NAME, CREATE_PASS) \ 2680 if (PassName == NAME) \ 2681 return true; 2682 #include "PassRegistry.def" 2683 return false; 2684 } 2685