1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a custom inliner that handles only functions that
10 // are marked as "always inline".
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/IPO/AlwaysInliner.h"
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/Analysis/AliasAnalysis.h"
17 #include "llvm/Analysis/AssumptionCache.h"
18 #include "llvm/Analysis/InlineCost.h"
19 #include "llvm/Analysis/ProfileSummaryInfo.h"
20 #include "llvm/Analysis/TargetLibraryInfo.h"
21 #include "llvm/IR/CallingConv.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/IR/Type.h"
26 #include "llvm/InitializePasses.h"
27 #include "llvm/Transforms/IPO.h"
28 #include "llvm/Transforms/IPO/Inliner.h"
29 #include "llvm/Transforms/Utils/Cloning.h"
30 #include "llvm/Transforms/Utils/ModuleUtils.h"
31 
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "inline"
35 
36 PreservedAnalyses AlwaysInlinerPass::run(Module &M,
37                                          ModuleAnalysisManager &MAM) {
38   // Add inline assumptions during code generation.
39   FunctionAnalysisManager &FAM =
40       MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
41   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
42     return FAM.getResult<AssumptionAnalysis>(F);
43   };
44   auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
45 
46   SmallSetVector<CallBase *, 16> Calls;
47   bool Changed = false;
48   SmallVector<Function *, 16> InlinedFunctions;
49   for (Function &F : M) {
50     // When callee coroutine function is inlined into caller coroutine function
51     // before coro-split pass,
52     // coro-early pass can not handle this quiet well.
53     // So we won't inline the coroutine function if it have not been unsplited
54     if (F.isPresplitCoroutine())
55       continue;
56 
57     if (!F.isDeclaration() && isInlineViable(F).isSuccess()) {
58       Calls.clear();
59 
60       for (User *U : F.users())
61         if (auto *CB = dyn_cast<CallBase>(U))
62           if (CB->getCalledFunction() == &F &&
63                 CB->hasFnAttr(Attribute::AlwaysInline) &&
64                 !CB->getAttributes().hasFnAttr(Attribute::NoInline))
65               Calls.insert(CB);
66 
67       for (CallBase *CB : Calls) {
68         Function *Caller = CB->getCaller();
69         OptimizationRemarkEmitter ORE(Caller);
70         auto OIC = shouldInline(
71             *CB,
72             [&](CallBase &CB) {
73               return InlineCost::getAlways("always inline attribute");
74             },
75             ORE);
76         assert(OIC);
77         emitInlinedIntoBasedOnCost(ORE, CB->getDebugLoc(), CB->getParent(), F,
78                                    *Caller, *OIC, false, DEBUG_TYPE);
79 
80         InlineFunctionInfo IFI(
81             /*cg=*/nullptr, GetAssumptionCache, &PSI,
82             &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
83             &FAM.getResult<BlockFrequencyAnalysis>(F));
84 
85         InlineResult Res = InlineFunction(
86             *CB, IFI, &FAM.getResult<AAManager>(F), InsertLifetime);
87         assert(Res.isSuccess() && "unexpected failure to inline");
88         (void)Res;
89 
90         // Merge the attributes based on the inlining.
91         AttributeFuncs::mergeAttributesForInlining(*Caller, F);
92 
93         Changed = true;
94       }
95 
96       if (F.hasFnAttribute(Attribute::AlwaysInline)) {
97         // Remember to try and delete this function afterward. This both avoids
98         // re-walking the rest of the module and avoids dealing with any
99         // iterator invalidation issues while deleting functions.
100         InlinedFunctions.push_back(&F);
101       }
102     }
103   }
104 
105   // Remove any live functions.
106   erase_if(InlinedFunctions, [&](Function *F) {
107     F->removeDeadConstantUsers();
108     return !F->isDefTriviallyDead();
109   });
110 
111   // Delete the non-comdat ones from the module and also from our vector.
112   auto NonComdatBegin = partition(
113       InlinedFunctions, [&](Function *F) { return F->hasComdat(); });
114   for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) {
115     M.getFunctionList().erase(F);
116     Changed = true;
117   }
118   InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end());
119 
120   if (!InlinedFunctions.empty()) {
121     // Now we just have the comdat functions. Filter out the ones whose comdats
122     // are not actually dead.
123     filterDeadComdatFunctions(InlinedFunctions);
124     // The remaining functions are actually dead.
125     for (Function *F : InlinedFunctions) {
126       M.getFunctionList().erase(F);
127       Changed = true;
128     }
129   }
130 
131   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
132 }
133 
134 namespace {
135 
136 /// Inliner pass which only handles "always inline" functions.
137 ///
138 /// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner
139 /// base class to provide several facilities such as array alloca merging.
140 class AlwaysInlinerLegacyPass : public LegacyInlinerBase {
141 
142 public:
143   AlwaysInlinerLegacyPass() : LegacyInlinerBase(ID, /*InsertLifetime*/ true) {
144     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
145   }
146 
147   AlwaysInlinerLegacyPass(bool InsertLifetime)
148       : LegacyInlinerBase(ID, InsertLifetime) {
149     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
150   }
151 
152   /// Main run interface method.  We override here to avoid calling skipSCC().
153   bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); }
154 
155   static char ID; // Pass identification, replacement for typeid
156 
157   InlineCost getInlineCost(CallBase &CB) override;
158 
159   using llvm::Pass::doFinalization;
160   bool doFinalization(CallGraph &CG) override {
161     return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true);
162   }
163 };
164 }
165 
166 char AlwaysInlinerLegacyPass::ID = 0;
167 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
168                       "Inliner for always_inline functions", false, false)
169 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
170 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
171 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
172 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
173 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
174                     "Inliner for always_inline functions", false, false)
175 
176 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
177   return new AlwaysInlinerLegacyPass(InsertLifetime);
178 }
179 
180 /// Get the inline cost for the always-inliner.
181 ///
182 /// The always inliner *only* handles functions which are marked with the
183 /// attribute to force inlining. As such, it is dramatically simpler and avoids
184 /// using the powerful (but expensive) inline cost analysis. Instead it uses
185 /// a very simple and boring direct walk of the instructions looking for
186 /// impossible-to-inline constructs.
187 ///
188 /// Note, it would be possible to go to some lengths to cache the information
189 /// computed here, but as we only expect to do this for relatively few and
190 /// small functions which have the explicit attribute to force inlining, it is
191 /// likely not worth it in practice.
192 InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallBase &CB) {
193   Function *Callee = CB.getCalledFunction();
194 
195   // Only inline direct calls to functions with always-inline attributes
196   // that are viable for inlining.
197   if (!Callee)
198     return InlineCost::getNever("indirect call");
199 
200   // When callee coroutine function is inlined into caller coroutine function
201   // before coro-split pass,
202   // coro-early pass can not handle this quiet well.
203   // So we won't inline the coroutine function if it have not been unsplited
204   if (Callee->isPresplitCoroutine())
205     return InlineCost::getNever("unsplited coroutine call");
206 
207   // FIXME: We shouldn't even get here for declarations.
208   if (Callee->isDeclaration())
209     return InlineCost::getNever("no definition");
210 
211   if (!CB.hasFnAttr(Attribute::AlwaysInline))
212     return InlineCost::getNever("no alwaysinline attribute");
213 
214   if (Callee->hasFnAttribute(Attribute::AlwaysInline) && CB.isNoInline())
215     return InlineCost::getNever("noinline call site attribute");
216 
217   auto IsViable = isInlineViable(*Callee);
218   if (!IsViable.isSuccess())
219     return InlineCost::getNever(IsViable.getFailureReason());
220 
221   return InlineCost::getAlways("always inliner");
222 }
223