1 //===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass marks all internal functions as always_inline and creates
11 /// duplicates of all other functions and marks the duplicates as always_inline.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPU.h"
16 #include "AMDGPUTargetMachine.h"
17 #include "Utils/AMDGPUBaseInfo.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/Pass.h"
20 #include "llvm/Support/CommandLine.h"
21 
22 using namespace llvm;
23 
24 namespace {
25 
26 static cl::opt<bool> StressCalls(
27   "amdgpu-stress-function-calls",
28   cl::Hidden,
29   cl::desc("Force all functions to be noinline"),
30   cl::init(false));
31 
32 class AMDGPUAlwaysInline : public ModulePass {
33   bool GlobalOpt;
34 
35 public:
36   static char ID;
37 
38   AMDGPUAlwaysInline(bool GlobalOpt = false) :
39     ModulePass(ID), GlobalOpt(GlobalOpt) { }
40   bool runOnModule(Module &M) override;
41 
42   void getAnalysisUsage(AnalysisUsage &AU) const override {
43     AU.setPreservesAll();
44  }
45 };
46 
47 } // End anonymous namespace
48 
49 INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline",
50                 "AMDGPU Inline All Functions", false, false)
51 
52 char AMDGPUAlwaysInline::ID = 0;
53 
54 static void
55 recursivelyVisitUsers(GlobalValue &GV,
56                       SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {
57   SmallVector<User *, 16> Stack;
58 
59   SmallPtrSet<const Value *, 8> Visited;
60 
61   for (User *U : GV.users())
62     Stack.push_back(U);
63 
64   while (!Stack.empty()) {
65     User *U = Stack.pop_back_val();
66     if (!Visited.insert(U).second)
67       continue;
68 
69     if (Instruction *I = dyn_cast<Instruction>(U)) {
70       Function *F = I->getParent()->getParent();
71       if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
72         // FIXME: This is a horrible hack. We should always respect noinline,
73         // and just let us hit the error when we can't handle this.
74         //
75         // Unfortunately, clang adds noinline to all functions at -O0. We have
76         // to override this here. until that's fixed.
77         F->removeFnAttr(Attribute::NoInline);
78 
79         FuncsToAlwaysInline.insert(F);
80         Stack.push_back(F);
81       }
82 
83       // No need to look at further users, but we do need to inline any callers.
84       continue;
85     }
86 
87     for (User *UU : U->users())
88       Stack.push_back(UU);
89   }
90 }
91 
92 static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
93   std::vector<GlobalAlias*> AliasesToRemove;
94 
95   SmallPtrSet<Function *, 8> FuncsToAlwaysInline;
96   SmallPtrSet<Function *, 8> FuncsToNoInline;
97 
98   for (GlobalAlias &A : M.aliases()) {
99     if (Function* F = dyn_cast<Function>(A.getAliasee())) {
100       A.replaceAllUsesWith(F);
101       AliasesToRemove.push_back(&A);
102     }
103 
104     // FIXME: If the aliasee isn't a function, it's some kind of constant expr
105     // cast that won't be inlined through.
106   }
107 
108   if (GlobalOpt) {
109     for (GlobalAlias* A : AliasesToRemove) {
110       A->eraseFromParent();
111     }
112   }
113 
114   // Always force inlining of any function that uses an LDS global address. This
115   // is something of a workaround because we don't have a way of supporting LDS
116   // objects defined in functions. LDS is always allocated by a kernel, and it
117   // is difficult to manage LDS usage if a function may be used by multiple
118   // kernels.
119   //
120   // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
121   // should only appear when IPO passes manages to move LDs defined in a kernel
122   // into a single user function.
123 
124   for (GlobalVariable &GV : M.globals()) {
125     // TODO: Region address
126     unsigned AS = GV.getAddressSpace();
127     if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)
128       continue;
129 
130     recursivelyVisitUsers(GV, FuncsToAlwaysInline);
131   }
132 
133   if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
134     auto IncompatAttr
135       = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
136 
137     for (Function &F : M) {
138       if (!F.isDeclaration() && !F.use_empty() &&
139           !F.hasFnAttribute(IncompatAttr)) {
140         if (StressCalls) {
141           if (!FuncsToAlwaysInline.count(&F))
142             FuncsToNoInline.insert(&F);
143         } else
144           FuncsToAlwaysInline.insert(&F);
145       }
146     }
147   }
148 
149   for (Function *F : FuncsToAlwaysInline)
150     F->addFnAttr(Attribute::AlwaysInline);
151 
152   for (Function *F : FuncsToNoInline)
153     F->addFnAttr(Attribute::NoInline);
154 
155   return !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
156 }
157 
158 bool AMDGPUAlwaysInline::runOnModule(Module &M) {
159   return alwaysInlineImpl(M, GlobalOpt);
160 }
161 
162 ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {
163   return new AMDGPUAlwaysInline(GlobalOpt);
164 }
165 
166 PreservedAnalyses AMDGPUAlwaysInlinePass::run(Module &M,
167                                               ModuleAnalysisManager &AM) {
168   alwaysInlineImpl(M, GlobalOpt);
169   return PreservedAnalyses::all();
170 }
171