1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_increment intrinsics emitted by a frontend for
11 // profiling. It also builds the data structures and initialization code needed
12 // for updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/ProfileData/InstrProf.h"
17 #include "llvm/Transforms/Instrumentation.h"
18 
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Transforms/Utils/ModuleUtils.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "instrprof"
28 
29 namespace {
30 
31 class InstrProfiling : public ModulePass {
32 public:
33   static char ID;
34 
35   InstrProfiling() : ModulePass(ID) {}
36 
37   InstrProfiling(const InstrProfOptions &Options)
38       : ModulePass(ID), Options(Options) {}
39 
40   const char *getPassName() const override {
41     return "Frontend instrumentation-based coverage lowering";
42   }
43 
44   bool runOnModule(Module &M) override;
45 
46   void getAnalysisUsage(AnalysisUsage &AU) const override {
47     AU.setPreservesCFG();
48   }
49 
50 private:
51   InstrProfOptions Options;
52   Module *M;
53   DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
54   std::vector<Value *> UsedVars;
55 
56   bool isMachO() const {
57     return Triple(M->getTargetTriple()).isOSBinFormatMachO();
58   }
59 
60   /// Get the section name for the counter variables.
61   StringRef getCountersSection() const {
62     return getInstrProfCountersSectionName(isMachO());
63   }
64 
65   /// Get the section name for the name variables.
66   StringRef getNameSection() const {
67     return getInstrProfNameSectionName(isMachO());
68   }
69 
70   /// Get the section name for the profile data variables.
71   StringRef getDataSection() const {
72     return getInstrProfDataSectionName(isMachO());
73   }
74 
75   /// Get the section name for the coverage mapping data.
76   StringRef getCoverageSection() const {
77     return getInstrProfCoverageSectionName(isMachO());
78   }
79 
80   /// Replace instrprof_increment with an increment of the appropriate value.
81   void lowerIncrement(InstrProfIncrementInst *Inc);
82 
83   /// Set up the section and uses for coverage data and its references.
84   void lowerCoverageData(GlobalVariable *CoverageData);
85 
86   /// Get the region counters for an increment, creating them if necessary.
87   ///
88   /// If the counter array doesn't yet exist, the profile data variables
89   /// referring to them will also be created.
90   GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
91 
92   /// Emit runtime registration functions for each profile data variable.
93   void emitRegistration();
94 
95   /// Emit the necessary plumbing to pull in the runtime initialization.
96   void emitRuntimeHook();
97 
98   /// Add uses of our data variables and runtime hook.
99   void emitUses();
100 
101   /// Create a static initializer for our data, on platforms that need it,
102   /// and for any profile output file that was specified.
103   void emitInitialization();
104 };
105 
106 } // anonymous namespace
107 
108 char InstrProfiling::ID = 0;
109 INITIALIZE_PASS(InstrProfiling, "instrprof",
110                 "Frontend instrumentation-based coverage lowering.", false,
111                 false)
112 
113 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
114   return new InstrProfiling(Options);
115 }
116 
117 bool InstrProfiling::runOnModule(Module &M) {
118   bool MadeChange = false;
119 
120   this->M = &M;
121   RegionCounters.clear();
122   UsedVars.clear();
123 
124   for (Function &F : M)
125     for (BasicBlock &BB : F)
126       for (auto I = BB.begin(), E = BB.end(); I != E;)
127         if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
128           lowerIncrement(Inc);
129           MadeChange = true;
130         }
131   if (GlobalVariable *Coverage =
132           M.getNamedGlobal(getCoverageMappingVarName())) {
133     lowerCoverageData(Coverage);
134     MadeChange = true;
135   }
136   if (!MadeChange)
137     return false;
138 
139   emitRegistration();
140   emitRuntimeHook();
141   emitUses();
142   emitInitialization();
143   return true;
144 }
145 
146 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
147   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
148 
149   IRBuilder<> Builder(Inc);
150   uint64_t Index = Inc->getIndex()->getZExtValue();
151   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
152   Value *Count = Builder.CreateLoad(Addr, "pgocount");
153   Count = Builder.CreateAdd(Count, Builder.getInt64(1));
154   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
155   Inc->eraseFromParent();
156 }
157 
158 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
159   CoverageData->setSection(getCoverageSection());
160   CoverageData->setAlignment(8);
161 
162   Constant *Init = CoverageData->getInitializer();
163   // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
164   // for some C. If not, the frontend's given us something broken.
165   assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
166   assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
167          "invalid function list in coverage map");
168   ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
169   for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
170     Constant *Record = Records->getOperand(I);
171     Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
172 
173     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
174     GlobalVariable *Name = cast<GlobalVariable>(V);
175 
176     // If we have region counters for this name, we've already handled it.
177     auto It = RegionCounters.find(Name);
178     if (It != RegionCounters.end())
179       continue;
180 
181     // Move the name variable to the right section.
182     Name->setSection(getNameSection());
183     Name->setAlignment(1);
184   }
185 }
186 
187 /// Get the name of a profiling variable for a particular function.
188 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
189   auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
190   StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
191   return (Prefix + Name).str();
192 }
193 
194 GlobalVariable *
195 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
196   GlobalVariable *NamePtr = Inc->getName();
197   auto It = RegionCounters.find(NamePtr);
198   if (It != RegionCounters.end())
199     return It->second;
200 
201   // Move the name variable to the right section. Place them in a COMDAT group
202   // if the associated function is a COMDAT. This will make sure that
203   // only one copy of counters of the COMDAT function will be emitted after
204   // linking.
205   Function *Fn = Inc->getParent()->getParent();
206   Comdat *ProfileVarsComdat = nullptr;
207   if (Fn->hasComdat())
208     ProfileVarsComdat = M->getOrInsertComdat(
209         StringRef(getVarName(Inc, getInstrProfComdatPrefix())));
210   NamePtr->setSection(getNameSection());
211   NamePtr->setAlignment(1);
212   NamePtr->setComdat(ProfileVarsComdat);
213 
214   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
215   LLVMContext &Ctx = M->getContext();
216   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
217 
218   // Create the counters variable.
219   auto *CounterPtr =
220       new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
221                          Constant::getNullValue(CounterTy),
222                          getVarName(Inc, getInstrProfCountersVarPrefix()));
223   CounterPtr->setVisibility(NamePtr->getVisibility());
224   CounterPtr->setSection(getCountersSection());
225   CounterPtr->setAlignment(8);
226   CounterPtr->setComdat(ProfileVarsComdat);
227 
228   RegionCounters[Inc->getName()] = CounterPtr;
229 
230   // Create data variable.
231 
232   Type *DataTypes[] = {
233     #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
234     #include "llvm/ProfileData/InstrProfData.inc"
235   };
236   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
237 
238   Constant *DataVals[] = {
239     #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
240     #include "llvm/ProfileData/InstrProfData.inc"
241   };
242 
243   auto *Data = new GlobalVariable(*M, DataTy, true, NamePtr->getLinkage(),
244                                   ConstantStruct::get(DataTy, DataVals),
245                                   getVarName(Inc, getInstrProfDataVarPrefix()));
246   Data->setVisibility(NamePtr->getVisibility());
247   Data->setSection(getDataSection());
248   Data->setAlignment(8);
249   Data->setComdat(ProfileVarsComdat);
250 
251   // Mark the data variable as used so that it isn't stripped out.
252   UsedVars.push_back(Data);
253 
254   return CounterPtr;
255 }
256 
257 void InstrProfiling::emitRegistration() {
258   // Don't do this for Darwin.  compiler-rt uses linker magic.
259   if (Triple(M->getTargetTriple()).isOSDarwin())
260     return;
261 
262   // Use linker script magic to get data/cnts/name start/end.
263   if (Triple(M->getTargetTriple()).isOSLinux() ||
264       Triple(M->getTargetTriple()).isOSFreeBSD())
265     return;
266 
267   // Construct the function.
268   auto *VoidTy = Type::getVoidTy(M->getContext());
269   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
270   auto *RegisterFTy = FunctionType::get(VoidTy, false);
271   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
272                                      getInstrProfRegFuncsName(), M);
273   RegisterF->setUnnamedAddr(true);
274   if (Options.NoRedZone) RegisterF->addFnAttr(Attribute::NoRedZone);
275 
276   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
277   auto *RuntimeRegisterF =
278       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
279                        getInstrProfRegFuncName(), M);
280 
281   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
282   for (Value *Data : UsedVars)
283     IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
284   IRB.CreateRetVoid();
285 }
286 
287 void InstrProfiling::emitRuntimeHook() {
288 
289   // We expect the linker to be invoked with -u<hook_var> flag for linux,
290   // for which case there is no need to emit the user function.
291   if (Triple(M->getTargetTriple()).isOSLinux())
292     return;
293 
294   // If the module's provided its own runtime, we don't need to do anything.
295   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) return;
296 
297   // Declare an external variable that will pull in the runtime initialization.
298   auto *Int32Ty = Type::getInt32Ty(M->getContext());
299   auto *Var =
300       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
301                          nullptr, getInstrProfRuntimeHookVarName());
302 
303   // Make a function that uses it.
304   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
305                                 GlobalValue::LinkOnceODRLinkage,
306                                 getInstrProfRuntimeHookVarUseFuncName(), M);
307   User->addFnAttr(Attribute::NoInline);
308   if (Options.NoRedZone) User->addFnAttr(Attribute::NoRedZone);
309   User->setVisibility(GlobalValue::HiddenVisibility);
310 
311   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
312   auto *Load = IRB.CreateLoad(Var);
313   IRB.CreateRet(Load);
314 
315   // Mark the user variable as used so that it isn't stripped out.
316   UsedVars.push_back(User);
317 }
318 
319 void InstrProfiling::emitUses() {
320   if (UsedVars.empty())
321     return;
322 
323   GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
324   std::vector<Constant *> MergedVars;
325   if (LLVMUsed) {
326     // Collect the existing members of llvm.used.
327     ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
328     for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
329       MergedVars.push_back(Inits->getOperand(I));
330     LLVMUsed->eraseFromParent();
331   }
332 
333   Type *i8PTy = Type::getInt8PtrTy(M->getContext());
334   // Add uses for our data.
335   for (auto *Value : UsedVars)
336     MergedVars.push_back(
337         ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
338 
339   // Recreate llvm.used.
340   ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
341   LLVMUsed =
342       new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
343                          ConstantArray::get(ATy, MergedVars), "llvm.used");
344 
345   LLVMUsed->setSection("llvm.metadata");
346 }
347 
348 void InstrProfiling::emitInitialization() {
349   std::string InstrProfileOutput = Options.InstrProfileOutput;
350 
351   Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
352   if (!RegisterF && InstrProfileOutput.empty()) return;
353 
354   // Create the initialization function.
355   auto *VoidTy = Type::getVoidTy(M->getContext());
356   auto *F = Function::Create(FunctionType::get(VoidTy, false),
357                              GlobalValue::InternalLinkage,
358                              getInstrProfInitFuncName(), M);
359   F->setUnnamedAddr(true);
360   F->addFnAttr(Attribute::NoInline);
361   if (Options.NoRedZone) F->addFnAttr(Attribute::NoRedZone);
362 
363   // Add the basic block and the necessary calls.
364   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
365   if (RegisterF)
366     IRB.CreateCall(RegisterF, {});
367   if (!InstrProfileOutput.empty()) {
368     auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
369     auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
370     auto *SetNameF = Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
371                                       getInstrProfFileOverriderFuncName(), M);
372 
373     // Create variable for profile name.
374     Constant *ProfileNameConst =
375         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
376     GlobalVariable *ProfileName =
377         new GlobalVariable(*M, ProfileNameConst->getType(), true,
378                            GlobalValue::PrivateLinkage, ProfileNameConst);
379 
380     IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
381   }
382   IRB.CreateRetVoid();
383 
384   appendToGlobalCtors(*M, F, 0);
385 }
386