1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10 // related types.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Analysis/InlineAdvisor.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/AssumptionCache.h"
17 #include "llvm/Analysis/InlineCost.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/ProfileSummaryInfo.h"
20 #include "llvm/Analysis/ReplayInlineAdvisor.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
24 #include "llvm/IR/DebugInfoMetadata.h"
25 #include "llvm/IR/PassManager.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/raw_ostream.h"
28
29 using namespace llvm;
30 #define DEBUG_TYPE "inline"
31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
32 #define LLVM_HAVE_TF_AOT
33 #endif
34
35 // This weirdly named statistic tracks the number of times that, when attempting
36 // to inline a function A into B, we analyze the callers of B in order to see
37 // if those would be more profitable and blocked inline steps.
38 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
39
40 /// Flag to add inline messages as callsite attributes 'inline-remark'.
41 static cl::opt<bool>
42 InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
43 cl::Hidden,
44 cl::desc("Enable adding inline-remark attribute to"
45 " callsites processed by inliner but decided"
46 " to be not inlined"));
47
48 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
49 cl::Hidden,
50 cl::desc("Enable deferred inlining"));
51
52 // An integer used to limit the cost of inline deferral. The default negative
53 // number tells shouldBeDeferred to only take the secondary cost into account.
54 static cl::opt<int>
55 InlineDeferralScale("inline-deferral-scale",
56 cl::desc("Scale to limit the cost of inline deferral"),
57 cl::init(2), cl::Hidden);
58
59 static cl::opt<bool>
60 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
61 cl::desc("If true, annotate inline advisor remarks "
62 "with LTO and pass information."));
63
64 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
65
66 namespace {
67 using namespace llvm::ore;
68 class MandatoryInlineAdvice : public InlineAdvice {
69 public:
MandatoryInlineAdvice(InlineAdvisor * Advisor,CallBase & CB,OptimizationRemarkEmitter & ORE,bool IsInliningMandatory)70 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
71 OptimizationRemarkEmitter &ORE,
72 bool IsInliningMandatory)
73 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
74
75 private:
recordInliningWithCalleeDeletedImpl()76 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
77
recordInliningImpl()78 void recordInliningImpl() override {
79 if (IsInliningRecommended)
80 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
81 [&](OptimizationRemark &Remark) {
82 Remark << ": always inline attribute";
83 });
84 }
85
recordUnsuccessfulInliningImpl(const InlineResult & Result)86 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
87 if (IsInliningRecommended)
88 ORE.emit([&]() {
89 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
90 "NotInlined", DLoc, Block)
91 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
92 << NV("Caller", Caller)
93 << "': " << NV("Reason", Result.getFailureReason());
94 });
95 }
96
recordUnattemptedInliningImpl()97 void recordUnattemptedInliningImpl() override {
98 assert(!IsInliningRecommended && "Expected to attempt inlining");
99 }
100 };
101 } // namespace
102
recordUnsuccessfulInliningImpl(const InlineResult & Result)103 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
104 const InlineResult &Result) {
105 using namespace ore;
106 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
107 "; " + inlineCostStr(*OIC));
108 ORE.emit([&]() {
109 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
110 "NotInlined", DLoc, Block)
111 << "'" << NV("Callee", Callee) << "' is not inlined into '"
112 << NV("Caller", Caller)
113 << "': " << NV("Reason", Result.getFailureReason());
114 });
115 }
116
recordInliningWithCalleeDeletedImpl()117 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
118 if (EmitRemarks)
119 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
120 /* ForProfileContext= */ false,
121 Advisor->getAnnotatedInlinePassName());
122 }
123
recordInliningImpl()124 void DefaultInlineAdvice::recordInliningImpl() {
125 if (EmitRemarks)
126 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
127 /* ForProfileContext= */ false,
128 Advisor->getAnnotatedInlinePassName());
129 }
130
getDefaultInlineAdvice(CallBase & CB,FunctionAnalysisManager & FAM,const InlineParams & Params)131 llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice(
132 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
133 Function &Caller = *CB.getCaller();
134 ProfileSummaryInfo *PSI =
135 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
136 .getCachedResult<ProfileSummaryAnalysis>(
137 *CB.getParent()->getParent()->getParent());
138
139 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
140 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
141 return FAM.getResult<AssumptionAnalysis>(F);
142 };
143 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
144 return FAM.getResult<BlockFrequencyAnalysis>(F);
145 };
146 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
147 return FAM.getResult<TargetLibraryAnalysis>(F);
148 };
149
150 auto GetInlineCost = [&](CallBase &CB) {
151 Function &Callee = *CB.getCalledFunction();
152 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
153 bool RemarksEnabled =
154 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
155 DEBUG_TYPE);
156 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
157 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
158 };
159 return llvm::shouldInline(
160 CB, GetInlineCost, ORE,
161 Params.EnableDeferral.value_or(EnableInlineDeferral));
162 }
163
164 std::unique_ptr<InlineAdvice>
getAdviceImpl(CallBase & CB)165 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
166 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
167 return std::make_unique<DefaultInlineAdvice>(
168 this, CB, OIC,
169 FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));
170 }
171
InlineAdvice(InlineAdvisor * Advisor,CallBase & CB,OptimizationRemarkEmitter & ORE,bool IsInliningRecommended)172 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
173 OptimizationRemarkEmitter &ORE,
174 bool IsInliningRecommended)
175 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
176 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
177 IsInliningRecommended(IsInliningRecommended) {}
178
recordInlineStatsIfNeeded()179 void InlineAdvice::recordInlineStatsIfNeeded() {
180 if (Advisor->ImportedFunctionsStats)
181 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
182 }
183
recordInlining()184 void InlineAdvice::recordInlining() {
185 markRecorded();
186 recordInlineStatsIfNeeded();
187 recordInliningImpl();
188 }
189
recordInliningWithCalleeDeleted()190 void InlineAdvice::recordInliningWithCalleeDeleted() {
191 markRecorded();
192 recordInlineStatsIfNeeded();
193 recordInliningWithCalleeDeletedImpl();
194 }
195
196 AnalysisKey InlineAdvisorAnalysis::Key;
197
tryCreate(InlineParams Params,InliningAdvisorMode Mode,const ReplayInlinerSettings & ReplaySettings,InlineContext IC)198 bool InlineAdvisorAnalysis::Result::tryCreate(
199 InlineParams Params, InliningAdvisorMode Mode,
200 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
201 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
202 switch (Mode) {
203 case InliningAdvisorMode::Default:
204 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
205 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
206 // Restrict replay to default advisor, ML advisors are stateful so
207 // replay will need augmentations to interleave with them correctly.
208 if (!ReplaySettings.ReplayFile.empty()) {
209 Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),
210 std::move(Advisor), ReplaySettings,
211 /* EmitRemarks =*/true, IC);
212 }
213 break;
214 case InliningAdvisorMode::Development:
215 #ifdef LLVM_HAVE_TF_API
216 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
217 Advisor =
218 llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) {
219 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
220 return OIC.hasValue();
221 });
222 #endif
223 break;
224 case InliningAdvisorMode::Release:
225 #ifdef LLVM_HAVE_TF_AOT
226 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
227 Advisor = llvm::getReleaseModeAdvisor(M, MAM);
228 #endif
229 break;
230 }
231
232 return !!Advisor;
233 }
234
235 /// Return true if inlining of CB can block the caller from being
236 /// inlined which is proved to be more beneficial. \p IC is the
237 /// estimated inline cost associated with callsite \p CB.
238 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
239 /// caller if \p CB is suppressed for inlining.
240 static bool
shouldBeDeferred(Function * Caller,InlineCost IC,int & TotalSecondaryCost,function_ref<InlineCost (CallBase & CB)> GetInlineCost)241 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
242 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
243 // For now we only handle local or inline functions.
244 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
245 return false;
246 // If the cost of inlining CB is non-positive, it is not going to prevent the
247 // caller from being inlined into its callers and hence we don't need to
248 // defer.
249 if (IC.getCost() <= 0)
250 return false;
251 // Try to detect the case where the current inlining candidate caller (call
252 // it B) is a static or linkonce-ODR function and is an inlining candidate
253 // elsewhere, and the current candidate callee (call it C) is large enough
254 // that inlining it into B would make B too big to inline later. In these
255 // circumstances it may be best not to inline C into B, but to inline B into
256 // its callers.
257 //
258 // This only applies to static and linkonce-ODR functions because those are
259 // expected to be available for inlining in the translation units where they
260 // are used. Thus we will always have the opportunity to make local inlining
261 // decisions. Importantly the linkonce-ODR linkage covers inline functions
262 // and templates in C++.
263 //
264 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
265 // the internal implementation of the inline cost metrics rather than
266 // treating them as truly abstract units etc.
267 TotalSecondaryCost = 0;
268 // The candidate cost to be imposed upon the current function.
269 int CandidateCost = IC.getCost() - 1;
270 // If the caller has local linkage and can be inlined to all its callers, we
271 // can apply a huge negative bonus to TotalSecondaryCost.
272 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
273 // This bool tracks what happens if we DO inline C into B.
274 bool InliningPreventsSomeOuterInline = false;
275 unsigned NumCallerUsers = 0;
276 for (User *U : Caller->users()) {
277 CallBase *CS2 = dyn_cast<CallBase>(U);
278
279 // If this isn't a call to Caller (it could be some other sort
280 // of reference) skip it. Such references will prevent the caller
281 // from being removed.
282 if (!CS2 || CS2->getCalledFunction() != Caller) {
283 ApplyLastCallBonus = false;
284 continue;
285 }
286
287 InlineCost IC2 = GetInlineCost(*CS2);
288 ++NumCallerCallersAnalyzed;
289 if (!IC2) {
290 ApplyLastCallBonus = false;
291 continue;
292 }
293 if (IC2.isAlways())
294 continue;
295
296 // See if inlining of the original callsite would erase the cost delta of
297 // this callsite. We subtract off the penalty for the call instruction,
298 // which we would be deleting.
299 if (IC2.getCostDelta() <= CandidateCost) {
300 InliningPreventsSomeOuterInline = true;
301 TotalSecondaryCost += IC2.getCost();
302 NumCallerUsers++;
303 }
304 }
305
306 if (!InliningPreventsSomeOuterInline)
307 return false;
308
309 // If all outer calls to Caller would get inlined, the cost for the last
310 // one is set very low by getInlineCost, in anticipation that Caller will
311 // be removed entirely. We did not account for this above unless there
312 // is only one caller of Caller.
313 if (ApplyLastCallBonus)
314 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
315
316 // If InlineDeferralScale is negative, then ignore the cost of primary
317 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
318 if (InlineDeferralScale < 0)
319 return TotalSecondaryCost < IC.getCost();
320
321 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
322 int Allowance = IC.getCost() * InlineDeferralScale;
323 return TotalCost < Allowance;
324 }
325
326 namespace llvm {
operator <<(raw_ostream & R,const ore::NV & Arg)327 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
328 return R << Arg.Val;
329 }
330
331 template <class RemarkT>
operator <<(RemarkT && R,const InlineCost & IC)332 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
333 using namespace ore;
334 if (IC.isAlways()) {
335 R << "(cost=always)";
336 } else if (IC.isNever()) {
337 R << "(cost=never)";
338 } else {
339 R << "(cost=" << ore::NV("Cost", IC.getCost())
340 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
341 }
342 if (const char *Reason = IC.getReason())
343 R << ": " << ore::NV("Reason", Reason);
344 return R;
345 }
346 } // namespace llvm
347
inlineCostStr(const InlineCost & IC)348 std::string llvm::inlineCostStr(const InlineCost &IC) {
349 std::string Buffer;
350 raw_string_ostream Remark(Buffer);
351 Remark << IC;
352 return Remark.str();
353 }
354
setInlineRemark(CallBase & CB,StringRef Message)355 void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
356 if (!InlineRemarkAttribute)
357 return;
358
359 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
360 CB.addFnAttr(Attr);
361 }
362
363 /// Return the cost only if the inliner should attempt to inline at the given
364 /// CallSite. If we return the cost, we will emit an optimisation remark later
365 /// using that cost, so we won't do so from this function. Return None if
366 /// inlining should not be attempted.
367 Optional<InlineCost>
shouldInline(CallBase & CB,function_ref<InlineCost (CallBase & CB)> GetInlineCost,OptimizationRemarkEmitter & ORE,bool EnableDeferral)368 llvm::shouldInline(CallBase &CB,
369 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
370 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
371 using namespace ore;
372
373 InlineCost IC = GetInlineCost(CB);
374 Instruction *Call = &CB;
375 Function *Callee = CB.getCalledFunction();
376 Function *Caller = CB.getCaller();
377
378 if (IC.isAlways()) {
379 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
380 << ", Call: " << CB << "\n");
381 return IC;
382 }
383
384 if (!IC) {
385 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
386 << ", Call: " << CB << "\n");
387 if (IC.isNever()) {
388 ORE.emit([&]() {
389 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
390 << "'" << NV("Callee", Callee) << "' not inlined into '"
391 << NV("Caller", Caller)
392 << "' because it should never be inlined " << IC;
393 });
394 } else {
395 ORE.emit([&]() {
396 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
397 << "'" << NV("Callee", Callee) << "' not inlined into '"
398 << NV("Caller", Caller) << "' because too costly to inline "
399 << IC;
400 });
401 }
402 setInlineRemark(CB, inlineCostStr(IC));
403 return None;
404 }
405
406 int TotalSecondaryCost = 0;
407 if (EnableDeferral &&
408 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
409 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
410 << " Cost = " << IC.getCost()
411 << ", outer Cost = " << TotalSecondaryCost << '\n');
412 ORE.emit([&]() {
413 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
414 Call)
415 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
416 << "' increases the cost of inlining '" << NV("Caller", Caller)
417 << "' in other contexts";
418 });
419 setInlineRemark(CB, "deferred");
420 return None;
421 }
422
423 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
424 << '\n');
425 return IC;
426 }
427
formatCallSiteLocation(DebugLoc DLoc,const CallSiteFormat & Format)428 std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
429 const CallSiteFormat &Format) {
430 std::string Buffer;
431 raw_string_ostream CallSiteLoc(Buffer);
432 bool First = true;
433 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
434 if (!First)
435 CallSiteLoc << " @ ";
436 // Note that negative line offset is actually possible, but we use
437 // unsigned int to match line offset representation in remarks so
438 // it's directly consumable by relay advisor.
439 uint32_t Offset =
440 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
441 uint32_t Discriminator = DIL->getBaseDiscriminator();
442 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
443 if (Name.empty())
444 Name = DIL->getScope()->getSubprogram()->getName();
445 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
446 if (Format.outputColumn())
447 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
448 if (Format.outputDiscriminator() && Discriminator)
449 CallSiteLoc << "." << llvm::utostr(Discriminator);
450 First = false;
451 }
452
453 return CallSiteLoc.str();
454 }
455
addLocationToRemarks(OptimizationRemark & Remark,DebugLoc DLoc)456 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
457 if (!DLoc) {
458 return;
459 }
460
461 bool First = true;
462 Remark << " at callsite ";
463 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
464 if (!First)
465 Remark << " @ ";
466 unsigned int Offset = DIL->getLine();
467 Offset -= DIL->getScope()->getSubprogram()->getLine();
468 unsigned int Discriminator = DIL->getBaseDiscriminator();
469 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
470 if (Name.empty())
471 Name = DIL->getScope()->getSubprogram()->getName();
472 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
473 << ore::NV("Column", DIL->getColumn());
474 if (Discriminator)
475 Remark << "." << ore::NV("Disc", Discriminator);
476 First = false;
477 }
478
479 Remark << ";";
480 }
481
emitInlinedInto(OptimizationRemarkEmitter & ORE,DebugLoc DLoc,const BasicBlock * Block,const Function & Callee,const Function & Caller,bool AlwaysInline,function_ref<void (OptimizationRemark &)> ExtraContext,const char * PassName)482 void llvm::emitInlinedInto(
483 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
484 const Function &Callee, const Function &Caller, bool AlwaysInline,
485 function_ref<void(OptimizationRemark &)> ExtraContext,
486 const char *PassName) {
487 ORE.emit([&]() {
488 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
489 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
490 DLoc, Block);
491 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
492 << ore::NV("Caller", &Caller) << "'";
493 if (ExtraContext)
494 ExtraContext(Remark);
495 addLocationToRemarks(Remark, DLoc);
496 return Remark;
497 });
498 }
499
emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter & ORE,DebugLoc DLoc,const BasicBlock * Block,const Function & Callee,const Function & Caller,const InlineCost & IC,bool ForProfileContext,const char * PassName)500 void llvm::emitInlinedIntoBasedOnCost(
501 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
502 const Function &Callee, const Function &Caller, const InlineCost &IC,
503 bool ForProfileContext, const char *PassName) {
504 llvm::emitInlinedInto(
505 ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
506 [&](OptimizationRemark &Remark) {
507 if (ForProfileContext)
508 Remark << " to match profiling context";
509 Remark << " with " << IC;
510 },
511 PassName);
512 }
513
InlineAdvisor(Module & M,FunctionAnalysisManager & FAM,Optional<InlineContext> IC)514 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
515 Optional<InlineContext> IC)
516 : M(M), FAM(FAM), IC(IC),
517 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
518 ? llvm::AnnotateInlinePassName(*IC)
519 : DEBUG_TYPE) {
520 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
521 ImportedFunctionsStats =
522 std::make_unique<ImportedFunctionsInliningStatistics>();
523 ImportedFunctionsStats->setModuleInfo(M);
524 }
525 }
526
~InlineAdvisor()527 InlineAdvisor::~InlineAdvisor() {
528 if (ImportedFunctionsStats) {
529 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
530 ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
531 InlinerFunctionImportStatsOpts::Verbose);
532 }
533 }
534
getMandatoryAdvice(CallBase & CB,bool Advice)535 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
536 bool Advice) {
537 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
538 Advice);
539 }
540
getLTOPhase(ThinOrFullLTOPhase LTOPhase)541 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
542 switch (LTOPhase) {
543 case (ThinOrFullLTOPhase::None):
544 return "main";
545 case (ThinOrFullLTOPhase::ThinLTOPreLink):
546 case (ThinOrFullLTOPhase::FullLTOPreLink):
547 return "prelink";
548 case (ThinOrFullLTOPhase::ThinLTOPostLink):
549 case (ThinOrFullLTOPhase::FullLTOPostLink):
550 return "postlink";
551 }
552 llvm_unreachable("unreachable");
553 }
554
getInlineAdvisorContext(InlinePass IP)555 static inline const char *getInlineAdvisorContext(InlinePass IP) {
556 switch (IP) {
557 case (InlinePass::AlwaysInliner):
558 return "always-inline";
559 case (InlinePass::CGSCCInliner):
560 return "cgscc-inline";
561 case (InlinePass::EarlyInliner):
562 return "early-inline";
563 case (InlinePass::MLInliner):
564 return "ml-inline";
565 case (InlinePass::ModuleInliner):
566 return "module-inline";
567 case (InlinePass::ReplayCGSCCInliner):
568 return "replay-cgscc-inline";
569 case (InlinePass::ReplaySampleProfileInliner):
570 return "replay-sample-profile-inline";
571 case (InlinePass::SampleProfileInliner):
572 return "sample-profile-inline";
573 }
574
575 llvm_unreachable("unreachable");
576 }
577
AnnotateInlinePassName(InlineContext IC)578 std::string llvm::AnnotateInlinePassName(InlineContext IC) {
579 return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
580 std::string(getInlineAdvisorContext(IC.Pass));
581 }
582
583 InlineAdvisor::MandatoryInliningKind
getMandatoryKind(CallBase & CB,FunctionAnalysisManager & FAM,OptimizationRemarkEmitter & ORE)584 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
585 OptimizationRemarkEmitter &ORE) {
586 auto &Callee = *CB.getCalledFunction();
587
588 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
589 return FAM.getResult<TargetLibraryAnalysis>(F);
590 };
591
592 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
593
594 auto TrivialDecision =
595 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
596
597 if (TrivialDecision) {
598 if (TrivialDecision->isSuccess())
599 return MandatoryInliningKind::Always;
600 else
601 return MandatoryInliningKind::Never;
602 }
603 return MandatoryInliningKind::NotMandatory;
604 }
605
getAdvice(CallBase & CB,bool MandatoryOnly)606 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
607 bool MandatoryOnly) {
608 if (!MandatoryOnly)
609 return getAdviceImpl(CB);
610 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
611 MandatoryInliningKind::Always ==
612 getMandatoryKind(CB, FAM, getCallerORE(CB));
613 return getMandatoryAdvice(CB, Advice);
614 }
615
getCallerORE(CallBase & CB)616 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
617 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());
618 }
619
620 PreservedAnalyses
run(Module & M,ModuleAnalysisManager & MAM)621 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
622 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
623 if (!IA)
624 OS << "No Inline Advisor\n";
625 else
626 IA->getAdvisor()->print(OS);
627 return PreservedAnalyses::all();
628 }
629
run(LazyCallGraph::SCC & InitialC,CGSCCAnalysisManager & AM,LazyCallGraph & CG,CGSCCUpdateResult & UR)630 PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
631 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
632 CGSCCUpdateResult &UR) {
633 const auto &MAMProxy =
634 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
635
636 if (InitialC.size() == 0) {
637 OS << "SCC is empty!\n";
638 return PreservedAnalyses::all();
639 }
640 Module &M = *InitialC.begin()->getFunction().getParent();
641 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
642 if (!IA)
643 OS << "No Inline Advisor\n";
644 else
645 IA->getAdvisor()->print(OS);
646 return PreservedAnalyses::all();
647 }
648