1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_increment intrinsics emitted by a frontend for
11 // profiling. It also builds the data structures and initialization code needed
12 // for updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/Transforms/Instrumentation.h"
17 
18 #include "llvm/ADT/Triple.h"
19 #include "llvm/IR/IRBuilder.h"
20 #include "llvm/IR/IntrinsicInst.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
23 
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "instrprof"
27 
28 namespace {
29 
30 class InstrProfiling : public ModulePass {
31 public:
32   static char ID;
33 
34   InstrProfiling() : ModulePass(ID) {}
35 
36   InstrProfiling(const InstrProfOptions &Options)
37       : ModulePass(ID), Options(Options) {}
38 
39   const char *getPassName() const override {
40     return "Frontend instrumentation-based coverage lowering";
41   }
42 
43   bool runOnModule(Module &M) override;
44 
45   void getAnalysisUsage(AnalysisUsage &AU) const override {
46     AU.setPreservesCFG();
47   }
48 
49 private:
50   InstrProfOptions Options;
51   Module *M;
52   DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
53   std::vector<Value *> UsedVars;
54 
55   bool isMachO() const {
56     return Triple(M->getTargetTriple()).isOSBinFormatMachO();
57   }
58 
59   /// Get the section name for the counter variables.
60   StringRef getCountersSection() const {
61     return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
62   }
63 
64   /// Get the section name for the name variables.
65   StringRef getNameSection() const {
66     return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
67   }
68 
69   /// Get the section name for the profile data variables.
70   StringRef getDataSection() const {
71     return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
72   }
73 
74   /// Get the section name for the coverage mapping data.
75   StringRef getCoverageSection() const {
76     return isMachO() ? "__DATA,__llvm_covmap" : "__llvm_covmap";
77   }
78 
79   /// Replace instrprof_increment with an increment of the appropriate value.
80   void lowerIncrement(InstrProfIncrementInst *Inc);
81 
82   /// Set up the section and uses for coverage data and its references.
83   void lowerCoverageData(GlobalVariable *CoverageData);
84 
85   /// Get the region counters for an increment, creating them if necessary.
86   ///
87   /// If the counter array doesn't yet exist, the profile data variables
88   /// referring to them will also be created.
89   GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
90 
91   /// Emit runtime registration functions for each profile data variable.
92   void emitRegistration();
93 
94   /// Emit the necessary plumbing to pull in the runtime initialization.
95   void emitRuntimeHook();
96 
97   /// Add uses of our data variables and runtime hook.
98   void emitUses();
99 
100   /// Create a static initializer for our data, on platforms that need it,
101   /// and for any profile output file that was specified.
102   void emitInitialization();
103 };
104 
105 } // anonymous namespace
106 
107 char InstrProfiling::ID = 0;
108 INITIALIZE_PASS(InstrProfiling, "instrprof",
109                 "Frontend instrumentation-based coverage lowering.", false,
110                 false)
111 
112 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
113   return new InstrProfiling(Options);
114 }
115 
116 bool InstrProfiling::runOnModule(Module &M) {
117   bool MadeChange = false;
118 
119   this->M = &M;
120   RegionCounters.clear();
121   UsedVars.clear();
122 
123   for (Function &F : M)
124     for (BasicBlock &BB : F)
125       for (auto I = BB.begin(), E = BB.end(); I != E;)
126         if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
127           lowerIncrement(Inc);
128           MadeChange = true;
129         }
130   if (GlobalVariable *Coverage = M.getNamedGlobal("__llvm_coverage_mapping")) {
131     lowerCoverageData(Coverage);
132     MadeChange = true;
133   }
134   if (!MadeChange)
135     return false;
136 
137   emitRegistration();
138   emitRuntimeHook();
139   emitUses();
140   emitInitialization();
141   return true;
142 }
143 
144 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
145   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
146 
147   IRBuilder<> Builder(Inc);
148   uint64_t Index = Inc->getIndex()->getZExtValue();
149   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
150   Value *Count = Builder.CreateLoad(Addr, "pgocount");
151   Count = Builder.CreateAdd(Count, Builder.getInt64(1));
152   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
153   Inc->eraseFromParent();
154 }
155 
156 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
157   CoverageData->setSection(getCoverageSection());
158   CoverageData->setAlignment(8);
159 
160   Constant *Init = CoverageData->getInitializer();
161   // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
162   // for some C. If not, the frontend's given us something broken.
163   assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
164   assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
165          "invalid function list in coverage map");
166   ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
167   for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
168     Constant *Record = Records->getOperand(I);
169     Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
170 
171     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
172     GlobalVariable *Name = cast<GlobalVariable>(V);
173 
174     // If we have region counters for this name, we've already handled it.
175     auto It = RegionCounters.find(Name);
176     if (It != RegionCounters.end())
177       continue;
178 
179     // Move the name variable to the right section.
180     Name->setSection(getNameSection());
181     Name->setAlignment(1);
182   }
183 }
184 
185 /// Get the name of a profiling variable for a particular function.
186 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
187   auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
188   StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
189   return ("__llvm_profile_" + VarName + "_" + Name).str();
190 }
191 
192 GlobalVariable *
193 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
194   GlobalVariable *Name = Inc->getName();
195   auto It = RegionCounters.find(Name);
196   if (It != RegionCounters.end())
197     return It->second;
198 
199   // Move the name variable to the right section. Place them in a COMDAT group
200   // if the associated function is a COMDAT. This will make sure that
201   // only one copy of counters of the COMDAT function will be emitted after
202   // linking.
203   Function *Fn = Inc->getParent()->getParent();
204   Comdat *ProfileVarsComdat = nullptr;
205   if (Fn->hasComdat())
206     ProfileVarsComdat = M->getOrInsertComdat(StringRef(getVarName(Inc, "vars")));
207   Name->setSection(getNameSection());
208   Name->setAlignment(1);
209   Name->setComdat(ProfileVarsComdat);
210 
211   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
212   LLVMContext &Ctx = M->getContext();
213   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
214 
215   // Create the counters variable.
216   auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
217                                       Constant::getNullValue(CounterTy),
218                                       getVarName(Inc, "counters"));
219   Counters->setVisibility(Name->getVisibility());
220   Counters->setSection(getCountersSection());
221   Counters->setAlignment(8);
222   Counters->setComdat(ProfileVarsComdat);
223 
224   RegionCounters[Inc->getName()] = Counters;
225 
226   // Create data variable.
227   auto *NameArrayTy = Name->getType()->getPointerElementType();
228   auto *Int32Ty = Type::getInt32Ty(Ctx);
229   auto *Int64Ty = Type::getInt64Ty(Ctx);
230   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
231   auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
232 
233   Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
234   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
235   Constant *DataVals[] = {
236       ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
237       ConstantInt::get(Int32Ty, NumCounters),
238       ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
239       ConstantExpr::getBitCast(Name, Int8PtrTy),
240       ConstantExpr::getBitCast(Counters, Int64PtrTy)};
241   auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
242                                   ConstantStruct::get(DataTy, DataVals),
243                                   getVarName(Inc, "data"));
244   Data->setVisibility(Name->getVisibility());
245   Data->setSection(getDataSection());
246   Data->setAlignment(8);
247   Data->setComdat(ProfileVarsComdat);
248 
249   // Mark the data variable as used so that it isn't stripped out.
250   UsedVars.push_back(Data);
251 
252   return Counters;
253 }
254 
255 void InstrProfiling::emitRegistration() {
256   // Don't do this for Darwin.  compiler-rt uses linker magic.
257   if (Triple(M->getTargetTriple()).isOSDarwin())
258     return;
259 
260   // Use linker script magic to get data/cnts/name start/end.
261   if (Triple(M->getTargetTriple()).isOSLinux() ||
262       Triple(M->getTargetTriple()).isOSFreeBSD())
263     return;
264 
265   // Construct the function.
266   auto *VoidTy = Type::getVoidTy(M->getContext());
267   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
268   auto *RegisterFTy = FunctionType::get(VoidTy, false);
269   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
270                                      "__llvm_profile_register_functions", M);
271   RegisterF->setUnnamedAddr(true);
272   if (Options.NoRedZone)
273     RegisterF->addFnAttr(Attribute::NoRedZone);
274 
275   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
276   auto *RuntimeRegisterF =
277       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
278                        "__llvm_profile_register_function", M);
279 
280   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
281   for (Value *Data : UsedVars)
282     IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
283   IRB.CreateRetVoid();
284 }
285 
286 void InstrProfiling::emitRuntimeHook() {
287   const char *const RuntimeVarName = "__llvm_profile_runtime";
288   const char *const RuntimeUserName = "__llvm_profile_runtime_user";
289 
290   // If the module's provided its own runtime, we don't need to do anything.
291   if (M->getGlobalVariable(RuntimeVarName))
292     return;
293 
294   // Declare an external variable that will pull in the runtime initialization.
295   auto *Int32Ty = Type::getInt32Ty(M->getContext());
296   auto *Var =
297       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
298                          nullptr, RuntimeVarName);
299 
300   // Make a function that uses it.
301   auto *User =
302       Function::Create(FunctionType::get(Int32Ty, false),
303                        GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
304   User->addFnAttr(Attribute::NoInline);
305   if (Options.NoRedZone)
306     User->addFnAttr(Attribute::NoRedZone);
307   User->setVisibility(GlobalValue::HiddenVisibility);
308 
309   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
310   auto *Load = IRB.CreateLoad(Var);
311   IRB.CreateRet(Load);
312 
313   // Mark the user variable as used so that it isn't stripped out.
314   UsedVars.push_back(User);
315 }
316 
317 void InstrProfiling::emitUses() {
318   if (UsedVars.empty())
319     return;
320 
321   GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
322   std::vector<Constant *> MergedVars;
323   if (LLVMUsed) {
324     // Collect the existing members of llvm.used.
325     ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
326     for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
327       MergedVars.push_back(Inits->getOperand(I));
328     LLVMUsed->eraseFromParent();
329   }
330 
331   Type *i8PTy = Type::getInt8PtrTy(M->getContext());
332   // Add uses for our data.
333   for (auto *Value : UsedVars)
334     MergedVars.push_back(
335         ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
336 
337   // Recreate llvm.used.
338   ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
339   LLVMUsed =
340       new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
341                          ConstantArray::get(ATy, MergedVars), "llvm.used");
342 
343   LLVMUsed->setSection("llvm.metadata");
344 }
345 
346 void InstrProfiling::emitInitialization() {
347   std::string InstrProfileOutput = Options.InstrProfileOutput;
348 
349   Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
350   if (!RegisterF && InstrProfileOutput.empty())
351     return;
352 
353   // Create the initialization function.
354   auto *VoidTy = Type::getVoidTy(M->getContext());
355   auto *F =
356       Function::Create(FunctionType::get(VoidTy, false),
357                        GlobalValue::InternalLinkage, "__llvm_profile_init", M);
358   F->setUnnamedAddr(true);
359   F->addFnAttr(Attribute::NoInline);
360   if (Options.NoRedZone)
361     F->addFnAttr(Attribute::NoRedZone);
362 
363   // Add the basic block and the necessary calls.
364   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
365   if (RegisterF)
366     IRB.CreateCall(RegisterF, {});
367   if (!InstrProfileOutput.empty()) {
368     auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
369     auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
370     auto *SetNameF =
371         Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
372                          "__llvm_profile_override_default_filename", M);
373 
374     // Create variable for profile name.
375     Constant *ProfileNameConst =
376         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
377     GlobalVariable *ProfileName =
378         new GlobalVariable(*M, ProfileNameConst->getType(), true,
379                            GlobalValue::PrivateLinkage, ProfileNameConst);
380 
381     IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
382   }
383   IRB.CreateRetVoid();
384 
385   appendToGlobalCtors(*M, F, 0);
386 }
387