1 //===- InlineAlways.cpp - Code to inline always_inline functions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a custom inliner that handles only functions that
10 // are marked as "always inline".
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/IPO/AlwaysInliner.h"
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/Analysis/AssumptionCache.h"
17 #include "llvm/Analysis/InlineCost.h"
18 #include "llvm/Analysis/ProfileSummaryInfo.h"
19 #include "llvm/Analysis/TargetLibraryInfo.h"
20 #include "llvm/IR/CallingConv.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/Module.h"
24 #include "llvm/IR/Type.h"
25 #include "llvm/InitializePasses.h"
26 #include "llvm/Transforms/IPO.h"
27 #include "llvm/Transforms/IPO/Inliner.h"
28 #include "llvm/Transforms/Utils/Cloning.h"
29 #include "llvm/Transforms/Utils/ModuleUtils.h"
30 
31 using namespace llvm;
32 
33 #define DEBUG_TYPE "inline"
34 
35 PreservedAnalyses AlwaysInlinerPass::run(Module &M,
36                                          ModuleAnalysisManager &MAM) {
37   // Add inline assumptions during code generation.
38   FunctionAnalysisManager &FAM =
39       MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
40   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
41     return FAM.getResult<AssumptionAnalysis>(F);
42   };
43   auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
44 
45   SmallSetVector<CallBase *, 16> Calls;
46   bool Changed = false;
47   SmallVector<Function *, 16> InlinedFunctions;
48   for (Function &F : M)
49     if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) &&
50         isInlineViable(F).isSuccess()) {
51       Calls.clear();
52 
53       for (User *U : F.users())
54         if (auto *CB = dyn_cast<CallBase>(U))
55           if (CB->getCalledFunction() == &F)
56             Calls.insert(CB);
57 
58       for (CallBase *CB : Calls) {
59         Function *Caller = CB->getCaller();
60         OptimizationRemarkEmitter ORE(Caller);
61         auto OIC = shouldInline(
62             *CB,
63             [&](CallBase &CB) {
64               return InlineCost::getAlways("always inline attribute");
65             },
66             ORE);
67         assert(OIC);
68         emitInlinedInto(ORE, CB->getDebugLoc(), CB->getParent(), F, *Caller,
69                         *OIC, false, DEBUG_TYPE);
70 
71         InlineFunctionInfo IFI(
72             /*cg=*/nullptr, GetAssumptionCache, &PSI,
73             &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
74             &FAM.getResult<BlockFrequencyAnalysis>(F));
75 
76         InlineResult Res =
77             InlineFunction(*CB, IFI, /*CalleeAAR=*/nullptr, InsertLifetime);
78         assert(Res.isSuccess() && "unexpected failure to inline");
79         (void)Res;
80         Changed = true;
81       }
82 
83       // Remember to try and delete this function afterward. This both avoids
84       // re-walking the rest of the module and avoids dealing with any iterator
85       // invalidation issues while deleting functions.
86       InlinedFunctions.push_back(&F);
87     }
88 
89   // Remove any live functions.
90   erase_if(InlinedFunctions, [&](Function *F) {
91     F->removeDeadConstantUsers();
92     return !F->isDefTriviallyDead();
93   });
94 
95   // Delete the non-comdat ones from the module and also from our vector.
96   auto NonComdatBegin = partition(
97       InlinedFunctions, [&](Function *F) { return F->hasComdat(); });
98   for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end()))
99     M.getFunctionList().erase(F);
100   InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end());
101 
102   if (!InlinedFunctions.empty()) {
103     // Now we just have the comdat functions. Filter out the ones whose comdats
104     // are not actually dead.
105     filterDeadComdatFunctions(M, InlinedFunctions);
106     // The remaining functions are actually dead.
107     for (Function *F : InlinedFunctions)
108       M.getFunctionList().erase(F);
109   }
110 
111   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
112 }
113 
114 namespace {
115 
116 /// Inliner pass which only handles "always inline" functions.
117 ///
118 /// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner
119 /// base class to provide several facilities such as array alloca merging.
120 class AlwaysInlinerLegacyPass : public LegacyInlinerBase {
121 
122 public:
123   AlwaysInlinerLegacyPass() : LegacyInlinerBase(ID, /*InsertLifetime*/ true) {
124     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
125   }
126 
127   AlwaysInlinerLegacyPass(bool InsertLifetime)
128       : LegacyInlinerBase(ID, InsertLifetime) {
129     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
130   }
131 
132   /// Main run interface method.  We override here to avoid calling skipSCC().
133   bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); }
134 
135   static char ID; // Pass identification, replacement for typeid
136 
137   InlineCost getInlineCost(CallBase &CB) override;
138 
139   using llvm::Pass::doFinalization;
140   bool doFinalization(CallGraph &CG) override {
141     return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true);
142   }
143 };
144 }
145 
146 char AlwaysInlinerLegacyPass::ID = 0;
147 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
148                       "Inliner for always_inline functions", false, false)
149 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
150 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
151 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
152 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
153 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
154                     "Inliner for always_inline functions", false, false)
155 
156 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
157   return new AlwaysInlinerLegacyPass(InsertLifetime);
158 }
159 
160 /// Get the inline cost for the always-inliner.
161 ///
162 /// The always inliner *only* handles functions which are marked with the
163 /// attribute to force inlining. As such, it is dramatically simpler and avoids
164 /// using the powerful (but expensive) inline cost analysis. Instead it uses
165 /// a very simple and boring direct walk of the instructions looking for
166 /// impossible-to-inline constructs.
167 ///
168 /// Note, it would be possible to go to some lengths to cache the information
169 /// computed here, but as we only expect to do this for relatively few and
170 /// small functions which have the explicit attribute to force inlining, it is
171 /// likely not worth it in practice.
172 InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallBase &CB) {
173   Function *Callee = CB.getCalledFunction();
174 
175   // Only inline direct calls to functions with always-inline attributes
176   // that are viable for inlining.
177   if (!Callee)
178     return InlineCost::getNever("indirect call");
179 
180   // FIXME: We shouldn't even get here for declarations.
181   if (Callee->isDeclaration())
182     return InlineCost::getNever("no definition");
183 
184   if (!CB.hasFnAttr(Attribute::AlwaysInline))
185     return InlineCost::getNever("no alwaysinline attribute");
186 
187   auto IsViable = isInlineViable(*Callee);
188   if (!IsViable.isSuccess())
189     return InlineCost::getNever(IsViable.getFailureReason());
190 
191   return InlineCost::getAlways("always inliner");
192 }
193