1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and 10 // related types. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Analysis/InlineAdvisor.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/AssumptionCache.h" 17 #include "llvm/Analysis/InlineCost.h" 18 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 19 #include "llvm/Analysis/ProfileSummaryInfo.h" 20 #include "llvm/Analysis/ReplayInlineAdvisor.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/Analysis/TargetTransformInfo.h" 23 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 24 #include "llvm/IR/DebugInfoMetadata.h" 25 #include "llvm/IR/PassManager.h" 26 #include "llvm/Support/CommandLine.h" 27 #include "llvm/Support/raw_ostream.h" 28 29 using namespace llvm; 30 #define DEBUG_TYPE "inline" 31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL 32 #define LLVM_HAVE_TF_AOT 33 #endif 34 35 // This weirdly named statistic tracks the number of times that, when attempting 36 // to inline a function A into B, we analyze the callers of B in order to see 37 // if those would be more profitable and blocked inline steps. 38 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed"); 39 40 /// Flag to add inline messages as callsite attributes 'inline-remark'. 41 static cl::opt<bool> 42 InlineRemarkAttribute("inline-remark-attribute", cl::init(false), 43 cl::Hidden, 44 cl::desc("Enable adding inline-remark attribute to" 45 " callsites processed by inliner but decided" 46 " to be not inlined")); 47 48 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false), 49 cl::Hidden, 50 cl::desc("Enable deferred inlining")); 51 52 // An integer used to limit the cost of inline deferral. The default negative 53 // number tells shouldBeDeferred to only take the secondary cost into account. 54 static cl::opt<int> 55 InlineDeferralScale("inline-deferral-scale", 56 cl::desc("Scale to limit the cost of inline deferral"), 57 cl::init(2), cl::Hidden); 58 59 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; 60 61 namespace { 62 using namespace llvm::ore; 63 class MandatoryInlineAdvice : public InlineAdvice { 64 public: 65 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 66 OptimizationRemarkEmitter &ORE, 67 bool IsInliningMandatory) 68 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {} 69 70 private: 71 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); } 72 73 void recordInliningImpl() override { 74 if (IsInliningRecommended) 75 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended, 76 [&](OptimizationRemark &Remark) { 77 Remark << ": always inline attribute"; 78 }); 79 } 80 81 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { 82 if (IsInliningRecommended) 83 ORE.emit([&]() { 84 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 85 "NotInlined", DLoc, Block) 86 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '" 87 << NV("Caller", Caller) 88 << "': " << NV("Reason", Result.getFailureReason()); 89 }); 90 } 91 92 void recordUnattemptedInliningImpl() override { 93 assert(!IsInliningRecommended && "Expected to attempt inlining"); 94 } 95 }; 96 } // namespace 97 98 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( 99 const InlineResult &Result) { 100 using namespace ore; 101 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) + 102 "; " + inlineCostStr(*OIC)); 103 ORE.emit([&]() { 104 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 105 "NotInlined", DLoc, Block) 106 << "'" << NV("Callee", Callee) << "' is not inlined into '" 107 << NV("Caller", Caller) 108 << "': " << NV("Reason", Result.getFailureReason()); 109 }); 110 } 111 112 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { 113 if (EmitRemarks) 114 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 115 /* ForProfileContext= */ false, 116 Advisor->getAnnotatedInlinePassName()); 117 } 118 119 void DefaultInlineAdvice::recordInliningImpl() { 120 if (EmitRemarks) 121 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 122 /* ForProfileContext= */ false, 123 Advisor->getAnnotatedInlinePassName()); 124 } 125 126 llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice( 127 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) { 128 Function &Caller = *CB.getCaller(); 129 ProfileSummaryInfo *PSI = 130 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller) 131 .getCachedResult<ProfileSummaryAnalysis>( 132 *CB.getParent()->getParent()->getParent()); 133 134 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller); 135 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 136 return FAM.getResult<AssumptionAnalysis>(F); 137 }; 138 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { 139 return FAM.getResult<BlockFrequencyAnalysis>(F); 140 }; 141 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 142 return FAM.getResult<TargetLibraryAnalysis>(F); 143 }; 144 145 auto GetInlineCost = [&](CallBase &CB) { 146 Function &Callee = *CB.getCalledFunction(); 147 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee); 148 bool RemarksEnabled = 149 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( 150 DEBUG_TYPE); 151 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI, 152 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr); 153 }; 154 return llvm::shouldInline( 155 CB, GetInlineCost, ORE, 156 Params.EnableDeferral.getValueOr(EnableInlineDeferral)); 157 } 158 159 std::unique_ptr<InlineAdvice> 160 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) { 161 auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 162 return std::make_unique<DefaultInlineAdvice>( 163 this, CB, OIC, 164 FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller())); 165 } 166 167 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 168 OptimizationRemarkEmitter &ORE, 169 bool IsInliningRecommended) 170 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()), 171 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE), 172 IsInliningRecommended(IsInliningRecommended) {} 173 174 void InlineAdvice::recordInlineStatsIfNeeded() { 175 if (Advisor->ImportedFunctionsStats) 176 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee); 177 } 178 179 void InlineAdvice::recordInlining() { 180 markRecorded(); 181 recordInlineStatsIfNeeded(); 182 recordInliningImpl(); 183 } 184 185 void InlineAdvice::recordInliningWithCalleeDeleted() { 186 markRecorded(); 187 recordInlineStatsIfNeeded(); 188 recordInliningWithCalleeDeletedImpl(); 189 } 190 191 AnalysisKey InlineAdvisorAnalysis::Key; 192 193 bool InlineAdvisorAnalysis::Result::tryCreate( 194 InlineParams Params, InliningAdvisorMode Mode, 195 const ReplayInlinerSettings &ReplaySettings) { 196 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 197 switch (Mode) { 198 case InliningAdvisorMode::Default: 199 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n"); 200 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params)); 201 // Restrict replay to default advisor, ML advisors are stateful so 202 // replay will need augmentations to interleave with them correctly. 203 if (!ReplaySettings.ReplayFile.empty()) { 204 Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), 205 std::move(Advisor), ReplaySettings, 206 /* EmitRemarks =*/true); 207 } 208 break; 209 case InliningAdvisorMode::Development: 210 #ifdef LLVM_HAVE_TF_API 211 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n"); 212 Advisor = 213 llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) { 214 auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 215 return OIC.hasValue(); 216 }); 217 #endif 218 break; 219 case InliningAdvisorMode::Release: 220 #ifdef LLVM_HAVE_TF_AOT 221 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n"); 222 Advisor = llvm::getReleaseModeAdvisor(M, MAM); 223 #endif 224 break; 225 } 226 227 return !!Advisor; 228 } 229 230 /// Return true if inlining of CB can block the caller from being 231 /// inlined which is proved to be more beneficial. \p IC is the 232 /// estimated inline cost associated with callsite \p CB. 233 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the 234 /// caller if \p CB is suppressed for inlining. 235 static bool 236 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, 237 function_ref<InlineCost(CallBase &CB)> GetInlineCost) { 238 // For now we only handle local or inline functions. 239 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) 240 return false; 241 // If the cost of inlining CB is non-positive, it is not going to prevent the 242 // caller from being inlined into its callers and hence we don't need to 243 // defer. 244 if (IC.getCost() <= 0) 245 return false; 246 // Try to detect the case where the current inlining candidate caller (call 247 // it B) is a static or linkonce-ODR function and is an inlining candidate 248 // elsewhere, and the current candidate callee (call it C) is large enough 249 // that inlining it into B would make B too big to inline later. In these 250 // circumstances it may be best not to inline C into B, but to inline B into 251 // its callers. 252 // 253 // This only applies to static and linkonce-ODR functions because those are 254 // expected to be available for inlining in the translation units where they 255 // are used. Thus we will always have the opportunity to make local inlining 256 // decisions. Importantly the linkonce-ODR linkage covers inline functions 257 // and templates in C++. 258 // 259 // FIXME: All of this logic should be sunk into getInlineCost. It relies on 260 // the internal implementation of the inline cost metrics rather than 261 // treating them as truly abstract units etc. 262 TotalSecondaryCost = 0; 263 // The candidate cost to be imposed upon the current function. 264 int CandidateCost = IC.getCost() - 1; 265 // If the caller has local linkage and can be inlined to all its callers, we 266 // can apply a huge negative bonus to TotalSecondaryCost. 267 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse(); 268 // This bool tracks what happens if we DO inline C into B. 269 bool InliningPreventsSomeOuterInline = false; 270 unsigned NumCallerUsers = 0; 271 for (User *U : Caller->users()) { 272 CallBase *CS2 = dyn_cast<CallBase>(U); 273 274 // If this isn't a call to Caller (it could be some other sort 275 // of reference) skip it. Such references will prevent the caller 276 // from being removed. 277 if (!CS2 || CS2->getCalledFunction() != Caller) { 278 ApplyLastCallBonus = false; 279 continue; 280 } 281 282 InlineCost IC2 = GetInlineCost(*CS2); 283 ++NumCallerCallersAnalyzed; 284 if (!IC2) { 285 ApplyLastCallBonus = false; 286 continue; 287 } 288 if (IC2.isAlways()) 289 continue; 290 291 // See if inlining of the original callsite would erase the cost delta of 292 // this callsite. We subtract off the penalty for the call instruction, 293 // which we would be deleting. 294 if (IC2.getCostDelta() <= CandidateCost) { 295 InliningPreventsSomeOuterInline = true; 296 TotalSecondaryCost += IC2.getCost(); 297 NumCallerUsers++; 298 } 299 } 300 301 if (!InliningPreventsSomeOuterInline) 302 return false; 303 304 // If all outer calls to Caller would get inlined, the cost for the last 305 // one is set very low by getInlineCost, in anticipation that Caller will 306 // be removed entirely. We did not account for this above unless there 307 // is only one caller of Caller. 308 if (ApplyLastCallBonus) 309 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus; 310 311 // If InlineDeferralScale is negative, then ignore the cost of primary 312 // inlining -- IC.getCost() multiplied by the number of callers to Caller. 313 if (InlineDeferralScale < 0) 314 return TotalSecondaryCost < IC.getCost(); 315 316 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers; 317 int Allowance = IC.getCost() * InlineDeferralScale; 318 return TotalCost < Allowance; 319 } 320 321 namespace llvm { 322 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) { 323 return R << Arg.Val; 324 } 325 326 template <class RemarkT> 327 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) { 328 using namespace ore; 329 if (IC.isAlways()) { 330 R << "(cost=always)"; 331 } else if (IC.isNever()) { 332 R << "(cost=never)"; 333 } else { 334 R << "(cost=" << ore::NV("Cost", IC.getCost()) 335 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")"; 336 } 337 if (const char *Reason = IC.getReason()) 338 R << ": " << ore::NV("Reason", Reason); 339 return R; 340 } 341 } // namespace llvm 342 343 std::string llvm::inlineCostStr(const InlineCost &IC) { 344 std::string Buffer; 345 raw_string_ostream Remark(Buffer); 346 Remark << IC; 347 return Remark.str(); 348 } 349 350 void llvm::setInlineRemark(CallBase &CB, StringRef Message) { 351 if (!InlineRemarkAttribute) 352 return; 353 354 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message); 355 CB.addFnAttr(Attr); 356 } 357 358 /// Return the cost only if the inliner should attempt to inline at the given 359 /// CallSite. If we return the cost, we will emit an optimisation remark later 360 /// using that cost, so we won't do so from this function. Return None if 361 /// inlining should not be attempted. 362 Optional<InlineCost> 363 llvm::shouldInline(CallBase &CB, 364 function_ref<InlineCost(CallBase &CB)> GetInlineCost, 365 OptimizationRemarkEmitter &ORE, bool EnableDeferral) { 366 using namespace ore; 367 368 InlineCost IC = GetInlineCost(CB); 369 Instruction *Call = &CB; 370 Function *Callee = CB.getCalledFunction(); 371 Function *Caller = CB.getCaller(); 372 373 if (IC.isAlways()) { 374 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) 375 << ", Call: " << CB << "\n"); 376 return IC; 377 } 378 379 if (!IC) { 380 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) 381 << ", Call: " << CB << "\n"); 382 if (IC.isNever()) { 383 ORE.emit([&]() { 384 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) 385 << "'" << NV("Callee", Callee) << "' not inlined into '" 386 << NV("Caller", Caller) 387 << "' because it should never be inlined " << IC; 388 }); 389 } else { 390 ORE.emit([&]() { 391 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) 392 << "'" << NV("Callee", Callee) << "' not inlined into '" 393 << NV("Caller", Caller) << "' because too costly to inline " 394 << IC; 395 }); 396 } 397 setInlineRemark(CB, inlineCostStr(IC)); 398 return None; 399 } 400 401 int TotalSecondaryCost = 0; 402 if (EnableDeferral && 403 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { 404 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB 405 << " Cost = " << IC.getCost() 406 << ", outer Cost = " << TotalSecondaryCost << '\n'); 407 ORE.emit([&]() { 408 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts", 409 Call) 410 << "Not inlining. Cost of inlining '" << NV("Callee", Callee) 411 << "' increases the cost of inlining '" << NV("Caller", Caller) 412 << "' in other contexts"; 413 }); 414 setInlineRemark(CB, "deferred"); 415 return None; 416 } 417 418 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB 419 << '\n'); 420 return IC; 421 } 422 423 std::string llvm::formatCallSiteLocation(DebugLoc DLoc, 424 const CallSiteFormat &Format) { 425 std::string Buffer; 426 raw_string_ostream CallSiteLoc(Buffer); 427 bool First = true; 428 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 429 if (!First) 430 CallSiteLoc << " @ "; 431 // Note that negative line offset is actually possible, but we use 432 // unsigned int to match line offset representation in remarks so 433 // it's directly consumable by relay advisor. 434 uint32_t Offset = 435 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine(); 436 uint32_t Discriminator = DIL->getBaseDiscriminator(); 437 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 438 if (Name.empty()) 439 Name = DIL->getScope()->getSubprogram()->getName(); 440 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset); 441 if (Format.outputColumn()) 442 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn()); 443 if (Format.outputDiscriminator() && Discriminator) 444 CallSiteLoc << "." << llvm::utostr(Discriminator); 445 First = false; 446 } 447 448 return CallSiteLoc.str(); 449 } 450 451 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) { 452 if (!DLoc) { 453 return; 454 } 455 456 bool First = true; 457 Remark << " at callsite "; 458 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 459 if (!First) 460 Remark << " @ "; 461 unsigned int Offset = DIL->getLine(); 462 Offset -= DIL->getScope()->getSubprogram()->getLine(); 463 unsigned int Discriminator = DIL->getBaseDiscriminator(); 464 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 465 if (Name.empty()) 466 Name = DIL->getScope()->getSubprogram()->getName(); 467 Remark << Name << ":" << ore::NV("Line", Offset) << ":" 468 << ore::NV("Column", DIL->getColumn()); 469 if (Discriminator) 470 Remark << "." << ore::NV("Disc", Discriminator); 471 First = false; 472 } 473 474 Remark << ";"; 475 } 476 477 void llvm::emitInlinedInto( 478 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 479 const Function &Callee, const Function &Caller, bool AlwaysInline, 480 function_ref<void(OptimizationRemark &)> ExtraContext, 481 const char *PassName) { 482 ORE.emit([&]() { 483 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined"; 484 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName, 485 DLoc, Block); 486 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '" 487 << ore::NV("Caller", &Caller) << "'"; 488 if (ExtraContext) 489 ExtraContext(Remark); 490 addLocationToRemarks(Remark, DLoc); 491 return Remark; 492 }); 493 } 494 495 void llvm::emitInlinedIntoBasedOnCost( 496 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 497 const Function &Callee, const Function &Caller, const InlineCost &IC, 498 bool ForProfileContext, const char *PassName) { 499 llvm::emitInlinedInto( 500 ORE, DLoc, Block, Callee, Caller, IC.isAlways(), 501 [&](OptimizationRemark &Remark) { 502 if (ForProfileContext) 503 Remark << " to match profiling context"; 504 Remark << " with " << IC; 505 }, 506 PassName); 507 } 508 509 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 510 Optional<InlineContext> IC) 511 : M(M), FAM(FAM), IC(IC) { 512 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { 513 ImportedFunctionsStats = 514 std::make_unique<ImportedFunctionsInliningStatistics>(); 515 ImportedFunctionsStats->setModuleInfo(M); 516 } 517 } 518 519 InlineAdvisor::~InlineAdvisor() { 520 if (ImportedFunctionsStats) { 521 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); 522 ImportedFunctionsStats->dump(InlinerFunctionImportStats == 523 InlinerFunctionImportStatsOpts::Verbose); 524 } 525 } 526 527 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB, 528 bool Advice) { 529 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB), 530 Advice); 531 } 532 533 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) { 534 switch (LTOPhase) { 535 case (ThinOrFullLTOPhase::None): 536 return "main"; 537 case (ThinOrFullLTOPhase::ThinLTOPreLink): 538 case (ThinOrFullLTOPhase::FullLTOPreLink): 539 return "prelink"; 540 case (ThinOrFullLTOPhase::ThinLTOPostLink): 541 case (ThinOrFullLTOPhase::FullLTOPostLink): 542 return "postlink"; 543 } 544 llvm_unreachable("unreachable"); 545 } 546 547 static inline const char *getInlineAdvisorContext(InlinePass IP) { 548 switch (IP) { 549 case (InlinePass::AlwaysInliner): 550 return "always-inline"; 551 case (InlinePass::CGSCCInliner): 552 return "cgscc-inline"; 553 case (InlinePass::EarlyInliner): 554 return "early-inline"; 555 case (InlinePass::MLInliner): 556 return "ml-inline"; 557 case (InlinePass::ModuleInliner): 558 return "module-inline"; 559 case (InlinePass::ReplayCGSCCInliner): 560 return "replay-cgscc-inline"; 561 case (InlinePass::ReplaySampleProfileInliner): 562 return "replay-sample-profile-inline"; 563 case (InlinePass::SampleProfileInliner): 564 return "sample-profile-inline"; 565 } 566 567 llvm_unreachable("unreachable"); 568 } 569 570 std::string llvm::AnnotateInlinePassName(InlineContext IC) { 571 return std::string(getLTOPhase(IC.LTOPhase)) + "-" + 572 std::string(getInlineAdvisorContext(IC.Pass)); 573 } 574 575 const char *InlineAdvisor::getAnnotatedInlinePassName() { 576 if (!IC.hasValue()) 577 return DEBUG_TYPE; 578 579 // IC is constant and initialized in constructor, so compute the annotated 580 // name only once. 581 static const std::string PassName = 582 llvm::AnnotateInlinePassName(IC.getValue()); 583 584 return PassName.c_str(); 585 } 586 587 InlineAdvisor::MandatoryInliningKind 588 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, 589 OptimizationRemarkEmitter &ORE) { 590 auto &Callee = *CB.getCalledFunction(); 591 592 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 593 return FAM.getResult<TargetLibraryAnalysis>(F); 594 }; 595 596 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee); 597 598 auto TrivialDecision = 599 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); 600 601 if (TrivialDecision.hasValue()) { 602 if (TrivialDecision->isSuccess()) 603 return MandatoryInliningKind::Always; 604 else 605 return MandatoryInliningKind::Never; 606 } 607 return MandatoryInliningKind::NotMandatory; 608 } 609 610 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB, 611 bool MandatoryOnly) { 612 if (!MandatoryOnly) 613 return getAdviceImpl(CB); 614 bool Advice = CB.getCaller() != CB.getCalledFunction() && 615 MandatoryInliningKind::Always == 616 getMandatoryKind(CB, FAM, getCallerORE(CB)); 617 return getMandatoryAdvice(CB, Advice); 618 } 619 620 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { 621 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()); 622 } 623 624 PreservedAnalyses 625 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) { 626 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M); 627 if (!IA) 628 OS << "No Inline Advisor\n"; 629 else 630 IA->getAdvisor()->print(OS); 631 return PreservedAnalyses::all(); 632 } 633