1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11 // It also builds the data structures and initialization code needed for
12 // updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/Transforms/InstrProfiling.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/IR/Attributes.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/Constant.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/GlobalValue.h"
30 #include "llvm/IR/GlobalVariable.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/IntrinsicInst.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Pass.h"
38 #include "llvm/ProfileData/InstrProf.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Error.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Transforms/Utils/ModuleUtils.h"
44 #include <algorithm>
45 #include <cassert>
46 #include <cstddef>
47 #include <cstdint>
48 #include <string>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "instrprof"
53 
54 // The start and end values of precise value profile range for memory
55 // intrinsic sizes
56 cl::opt<std::string> MemOPSizeRange(
57     "memop-size-range",
58     cl::desc("Set the range of size in memory intrinsic calls to be profiled "
59              "precisely, in a format of <start_val>:<end_val>"),
60     cl::init(""));
61 
62 // The value that considered to be large value in  memory intrinsic.
63 cl::opt<unsigned> MemOPSizeLarge(
64     "memop-size-large",
65     cl::desc("Set large value thresthold in memory intrinsic size profiling. "
66              "Value of 0 disables the large value profiling."),
67     cl::init(8192));
68 
69 namespace {
70 
71 cl::opt<bool> DoNameCompression("enable-name-compression",
72                                 cl::desc("Enable name string compression"),
73                                 cl::init(true));
74 
75 cl::opt<bool> DoHashBasedCounterSplit(
76     "hash-based-counter-split",
77     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
78     cl::init(true));
79 
80 cl::opt<bool> ValueProfileStaticAlloc(
81     "vp-static-alloc",
82     cl::desc("Do static counter allocation for value profiler"),
83     cl::init(true));
84 
85 cl::opt<double> NumCountersPerValueSite(
86     "vp-counters-per-site",
87     cl::desc("The average number of profile counters allocated "
88              "per value profiling site."),
89     // This is set to a very small value because in real programs, only
90     // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91     // For those sites with non-zero profile, the average number of targets
92     // is usually smaller than 2.
93     cl::init(1.0));
94 
95 class InstrProfilingLegacyPass : public ModulePass {
96   InstrProfiling InstrProf;
97 
98 public:
99   static char ID;
100 
101   InstrProfilingLegacyPass() : ModulePass(ID) {}
102   InstrProfilingLegacyPass(const InstrProfOptions &Options)
103       : ModulePass(ID), InstrProf(Options) {}
104 
105   StringRef getPassName() const override {
106     return "Frontend instrumentation-based coverage lowering";
107   }
108 
109   bool runOnModule(Module &M) override {
110     return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
111   }
112 
113   void getAnalysisUsage(AnalysisUsage &AU) const override {
114     AU.setPreservesCFG();
115     AU.addRequired<TargetLibraryInfoWrapperPass>();
116   }
117 };
118 
119 } // end anonymous namespace
120 
121 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
122   auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
123   if (!run(M, TLI))
124     return PreservedAnalyses::all();
125 
126   return PreservedAnalyses::none();
127 }
128 
129 char InstrProfilingLegacyPass::ID = 0;
130 INITIALIZE_PASS_BEGIN(
131     InstrProfilingLegacyPass, "instrprof",
132     "Frontend instrumentation-based coverage lowering.", false, false)
133 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
134 INITIALIZE_PASS_END(
135     InstrProfilingLegacyPass, "instrprof",
136     "Frontend instrumentation-based coverage lowering.", false, false)
137 
138 ModulePass *
139 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
140   return new InstrProfilingLegacyPass(Options);
141 }
142 
143 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
144   InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
145   if (Inc)
146     return Inc;
147   return dyn_cast<InstrProfIncrementInst>(Instr);
148 }
149 
150 bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
151   bool MadeChange = false;
152 
153   this->M = &M;
154   this->TLI = &TLI;
155   NamesVar = nullptr;
156   NamesSize = 0;
157   ProfileDataMap.clear();
158   UsedVars.clear();
159   getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
160                               MemOPSizeRangeLast);
161   TT = Triple(M.getTargetTriple());
162 
163   // We did not know how many value sites there would be inside
164   // the instrumented function. This is counting the number of instrumented
165   // target value sites to enter it as field in the profile data variable.
166   for (Function &F : M) {
167     InstrProfIncrementInst *FirstProfIncInst = nullptr;
168     for (BasicBlock &BB : F)
169       for (auto I = BB.begin(), E = BB.end(); I != E; I++)
170         if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
171           computeNumValueSiteCounts(Ind);
172         else if (FirstProfIncInst == nullptr)
173           FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
174 
175     // Value profiling intrinsic lowering requires per-function profile data
176     // variable to be created first.
177     if (FirstProfIncInst != nullptr)
178       static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
179   }
180 
181   for (Function &F : M)
182     for (BasicBlock &BB : F)
183       for (auto I = BB.begin(), E = BB.end(); I != E;) {
184         auto Instr = I++;
185         InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
186         if (Inc) {
187           lowerIncrement(Inc);
188           MadeChange = true;
189         } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
190           lowerValueProfileInst(Ind);
191           MadeChange = true;
192         }
193       }
194 
195   if (GlobalVariable *CoverageNamesVar =
196           M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
197     lowerCoverageData(CoverageNamesVar);
198     MadeChange = true;
199   }
200 
201   if (!MadeChange)
202     return false;
203 
204   emitVNodes();
205   emitNameData();
206   emitRegistration();
207   emitRuntimeHook();
208   emitUses();
209   emitInitialization();
210   return true;
211 }
212 
213 static Constant *getOrInsertValueProfilingCall(Module &M,
214                                                const TargetLibraryInfo &TLI,
215                                                bool IsRange = false) {
216   LLVMContext &Ctx = M.getContext();
217   auto *ReturnTy = Type::getVoidTy(M.getContext());
218 
219   Constant *Res;
220   if (!IsRange) {
221     Type *ParamTypes[] = {
222 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
223 #include "llvm/ProfileData/InstrProfData.inc"
224     };
225     auto *ValueProfilingCallTy =
226         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
227     Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
228                                 ValueProfilingCallTy);
229   } else {
230     Type *RangeParamTypes[] = {
231 #define VALUE_RANGE_PROF 1
232 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
233 #include "llvm/ProfileData/InstrProfData.inc"
234 #undef VALUE_RANGE_PROF
235     };
236     auto *ValueRangeProfilingCallTy =
237         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
238     Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
239                                 ValueRangeProfilingCallTy);
240   }
241 
242   if (Function *FunRes = dyn_cast<Function>(Res)) {
243     if (auto AK = TLI.getExtAttrForI32Param(false))
244       FunRes->addAttribute(3, AK);
245   }
246   return Res;
247 }
248 
249 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
250   GlobalVariable *Name = Ind->getName();
251   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
252   uint64_t Index = Ind->getIndex()->getZExtValue();
253   auto It = ProfileDataMap.find(Name);
254   if (It == ProfileDataMap.end()) {
255     PerFunctionProfileData PD;
256     PD.NumValueSites[ValueKind] = Index + 1;
257     ProfileDataMap[Name] = PD;
258   } else if (It->second.NumValueSites[ValueKind] <= Index)
259     It->second.NumValueSites[ValueKind] = Index + 1;
260 }
261 
262 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
263   GlobalVariable *Name = Ind->getName();
264   auto It = ProfileDataMap.find(Name);
265   assert(It != ProfileDataMap.end() && It->second.DataVar &&
266          "value profiling detected in function with no counter incerement");
267 
268   GlobalVariable *DataVar = It->second.DataVar;
269   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
270   uint64_t Index = Ind->getIndex()->getZExtValue();
271   for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
272     Index += It->second.NumValueSites[Kind];
273 
274   IRBuilder<> Builder(Ind);
275   bool IsRange = (Ind->getValueKind()->getZExtValue() ==
276                   llvm::InstrProfValueKind::IPVK_MemOPSize);
277   CallInst *Call = nullptr;
278   if (!IsRange) {
279     Value *Args[3] = {Ind->getTargetValue(),
280                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
281                       Builder.getInt32(Index)};
282     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
283   } else {
284     Value *Args[6] = {
285         Ind->getTargetValue(),
286         Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
287         Builder.getInt32(Index),
288         Builder.getInt64(MemOPSizeRangeStart),
289         Builder.getInt64(MemOPSizeRangeLast),
290         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
291     Call =
292         Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
293   }
294   if (auto AK = TLI->getExtAttrForI32Param(false))
295     Call->addAttribute(3, AK);
296   Ind->replaceAllUsesWith(Call);
297   Ind->eraseFromParent();
298 }
299 
300 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
301   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
302 
303   IRBuilder<> Builder(Inc);
304   uint64_t Index = Inc->getIndex()->getZExtValue();
305   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
306   Value *Count = Builder.CreateLoad(Addr, "pgocount");
307   Count = Builder.CreateAdd(Count, Inc->getStep());
308   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
309   Inc->eraseFromParent();
310 }
311 
312 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
313   ConstantArray *Names =
314       cast<ConstantArray>(CoverageNamesVar->getInitializer());
315   for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
316     Constant *NC = Names->getOperand(I);
317     Value *V = NC->stripPointerCasts();
318     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
319     GlobalVariable *Name = cast<GlobalVariable>(V);
320 
321     Name->setLinkage(GlobalValue::PrivateLinkage);
322     ReferencedNames.push_back(Name);
323     NC->dropAllReferences();
324   }
325   CoverageNamesVar->eraseFromParent();
326 }
327 
328 /// Get the name of a profiling variable for a particular function.
329 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
330   StringRef NamePrefix = getInstrProfNameVarPrefix();
331   StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
332   Function *F = Inc->getParent()->getParent();
333   Module *M = F->getParent();
334   if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
335       !canRenameComdatFunc(*F))
336     return (Prefix + Name).str();
337   uint64_t FuncHash = Inc->getHash()->getZExtValue();
338   SmallVector<char, 24> HashPostfix;
339   if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
340     return (Prefix + Name).str();
341   return (Prefix + Name + "." + Twine(FuncHash)).str();
342 }
343 
344 static inline bool shouldRecordFunctionAddr(Function *F) {
345   // Check the linkage
346   if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
347       !F->hasAvailableExternallyLinkage())
348     return true;
349   // Prohibit function address recording if the function is both internal and
350   // COMDAT. This avoids the profile data variable referencing internal symbols
351   // in COMDAT.
352   if (F->hasLocalLinkage() && F->hasComdat())
353     return false;
354   // Check uses of this function for other than direct calls or invokes to it.
355   // Inline virtual functions have linkeOnceODR linkage. When a key method
356   // exists, the vtable will only be emitted in the TU where the key method
357   // is defined. In a TU where vtable is not available, the function won't
358   // be 'addresstaken'. If its address is not recorded here, the profile data
359   // with missing address may be picked by the linker leading  to missing
360   // indirect call target info.
361   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
362 }
363 
364 static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
365                                                InstrProfIncrementInst *Inc) {
366   if (!needsComdatForCounter(F, M))
367     return nullptr;
368 
369   // COFF format requires a COMDAT section to have a key symbol with the same
370   // name. The linker targeting COFF also requires that the COMDAT
371   // a section is associated to must precede the associating section. For this
372   // reason, we must choose the counter var's name as the name of the comdat.
373   StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
374                                 ? getInstrProfCountersVarPrefix()
375                                 : getInstrProfComdatPrefix());
376   return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
377 }
378 
379 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
380   // Don't do this for Darwin.  compiler-rt uses linker magic.
381   if (Triple(M.getTargetTriple()).isOSDarwin())
382     return false;
383 
384   // Use linker script magic to get data/cnts/name start/end.
385   if (Triple(M.getTargetTriple()).isOSLinux() ||
386       Triple(M.getTargetTriple()).isOSFreeBSD() ||
387       Triple(M.getTargetTriple()).isPS4CPU())
388     return false;
389 
390   return true;
391 }
392 
393 GlobalVariable *
394 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
395   GlobalVariable *NamePtr = Inc->getName();
396   auto It = ProfileDataMap.find(NamePtr);
397   PerFunctionProfileData PD;
398   if (It != ProfileDataMap.end()) {
399     if (It->second.RegionCounters)
400       return It->second.RegionCounters;
401     PD = It->second;
402   }
403 
404   // Move the name variable to the right section. Place them in a COMDAT group
405   // if the associated function is a COMDAT. This will make sure that
406   // only one copy of counters of the COMDAT function will be emitted after
407   // linking.
408   Function *Fn = Inc->getParent()->getParent();
409   Comdat *ProfileVarsComdat = nullptr;
410   ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
411 
412   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
413   LLVMContext &Ctx = M->getContext();
414   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
415 
416   // Create the counters variable.
417   auto *CounterPtr =
418       new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
419                          Constant::getNullValue(CounterTy),
420                          getVarName(Inc, getInstrProfCountersVarPrefix()));
421   CounterPtr->setVisibility(NamePtr->getVisibility());
422   CounterPtr->setSection(
423       getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
424   CounterPtr->setAlignment(8);
425   CounterPtr->setComdat(ProfileVarsComdat);
426 
427   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
428   // Allocate statically the array of pointers to value profile nodes for
429   // the current function.
430   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
431   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
432     uint64_t NS = 0;
433     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
434       NS += PD.NumValueSites[Kind];
435     if (NS) {
436       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
437 
438       auto *ValuesVar =
439           new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
440                              Constant::getNullValue(ValuesTy),
441                              getVarName(Inc, getInstrProfValuesVarPrefix()));
442       ValuesVar->setVisibility(NamePtr->getVisibility());
443       ValuesVar->setSection(
444           getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
445       ValuesVar->setAlignment(8);
446       ValuesVar->setComdat(ProfileVarsComdat);
447       ValuesPtrExpr =
448           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
449     }
450   }
451 
452   // Create data variable.
453   auto *Int16Ty = Type::getInt16Ty(Ctx);
454   auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
455   Type *DataTypes[] = {
456 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
457 #include "llvm/ProfileData/InstrProfData.inc"
458   };
459   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
460 
461   Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
462                                ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
463                                : ConstantPointerNull::get(Int8PtrTy);
464 
465   Constant *Int16ArrayVals[IPVK_Last + 1];
466   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
467     Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
468 
469   Constant *DataVals[] = {
470 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
471 #include "llvm/ProfileData/InstrProfData.inc"
472   };
473   auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
474                                   ConstantStruct::get(DataTy, DataVals),
475                                   getVarName(Inc, getInstrProfDataVarPrefix()));
476   Data->setVisibility(NamePtr->getVisibility());
477   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
478   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
479   Data->setComdat(ProfileVarsComdat);
480 
481   PD.RegionCounters = CounterPtr;
482   PD.DataVar = Data;
483   ProfileDataMap[NamePtr] = PD;
484 
485   // Mark the data variable as used so that it isn't stripped out.
486   UsedVars.push_back(Data);
487   // Now that the linkage set by the FE has been passed to the data and counter
488   // variables, reset Name variable's linkage and visibility to private so that
489   // it can be removed later by the compiler.
490   NamePtr->setLinkage(GlobalValue::PrivateLinkage);
491   // Collect the referenced names to be used by emitNameData.
492   ReferencedNames.push_back(NamePtr);
493 
494   return CounterPtr;
495 }
496 
497 void InstrProfiling::emitVNodes() {
498   if (!ValueProfileStaticAlloc)
499     return;
500 
501   // For now only support this on platforms that do
502   // not require runtime registration to discover
503   // named section start/end.
504   if (needsRuntimeRegistrationOfSectionRange(*M))
505     return;
506 
507   size_t TotalNS = 0;
508   for (auto &PD : ProfileDataMap) {
509     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
510       TotalNS += PD.second.NumValueSites[Kind];
511   }
512 
513   if (!TotalNS)
514     return;
515 
516   uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
517 // Heuristic for small programs with very few total value sites.
518 // The default value of vp-counters-per-site is chosen based on
519 // the observation that large apps usually have a low percentage
520 // of value sites that actually have any profile data, and thus
521 // the average number of counters per site is low. For small
522 // apps with very few sites, this may not be true. Bump up the
523 // number of counters in this case.
524 #define INSTR_PROF_MIN_VAL_COUNTS 10
525   if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
526     NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
527 
528   auto &Ctx = M->getContext();
529   Type *VNodeTypes[] = {
530 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
531 #include "llvm/ProfileData/InstrProfData.inc"
532   };
533   auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
534 
535   ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
536   auto *VNodesVar = new GlobalVariable(
537       *M, VNodesTy, false, GlobalValue::PrivateLinkage,
538       Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
539   VNodesVar->setSection(
540       getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
541   UsedVars.push_back(VNodesVar);
542 }
543 
544 void InstrProfiling::emitNameData() {
545   std::string UncompressedData;
546 
547   if (ReferencedNames.empty())
548     return;
549 
550   std::string CompressedNameStr;
551   if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
552                                           DoNameCompression)) {
553     report_fatal_error(toString(std::move(E)), false);
554   }
555 
556   auto &Ctx = M->getContext();
557   auto *NamesVal = ConstantDataArray::getString(
558       Ctx, StringRef(CompressedNameStr), false);
559   NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
560                                 GlobalValue::PrivateLinkage, NamesVal,
561                                 getInstrProfNamesVarName());
562   NamesSize = CompressedNameStr.size();
563   NamesVar->setSection(
564       getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
565   UsedVars.push_back(NamesVar);
566 
567   for (auto *NamePtr : ReferencedNames)
568     NamePtr->eraseFromParent();
569 }
570 
571 void InstrProfiling::emitRegistration() {
572   if (!needsRuntimeRegistrationOfSectionRange(*M))
573     return;
574 
575   // Construct the function.
576   auto *VoidTy = Type::getVoidTy(M->getContext());
577   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
578   auto *Int64Ty = Type::getInt64Ty(M->getContext());
579   auto *RegisterFTy = FunctionType::get(VoidTy, false);
580   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
581                                      getInstrProfRegFuncsName(), M);
582   RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
583   if (Options.NoRedZone)
584     RegisterF->addFnAttr(Attribute::NoRedZone);
585 
586   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
587   auto *RuntimeRegisterF =
588       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
589                        getInstrProfRegFuncName(), M);
590 
591   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
592   for (Value *Data : UsedVars)
593     if (Data != NamesVar)
594       IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
595 
596   if (NamesVar) {
597     Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
598     auto *NamesRegisterTy =
599         FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
600     auto *NamesRegisterF =
601         Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
602                          getInstrProfNamesRegFuncName(), M);
603     IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
604                                     IRB.getInt64(NamesSize)});
605   }
606 
607   IRB.CreateRetVoid();
608 }
609 
610 void InstrProfiling::emitRuntimeHook() {
611   // We expect the linker to be invoked with -u<hook_var> flag for linux,
612   // for which case there is no need to emit the user function.
613   if (Triple(M->getTargetTriple()).isOSLinux())
614     return;
615 
616   // If the module's provided its own runtime, we don't need to do anything.
617   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
618     return;
619 
620   // Declare an external variable that will pull in the runtime initialization.
621   auto *Int32Ty = Type::getInt32Ty(M->getContext());
622   auto *Var =
623       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
624                          nullptr, getInstrProfRuntimeHookVarName());
625 
626   // Make a function that uses it.
627   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
628                                 GlobalValue::LinkOnceODRLinkage,
629                                 getInstrProfRuntimeHookVarUseFuncName(), M);
630   User->addFnAttr(Attribute::NoInline);
631   if (Options.NoRedZone)
632     User->addFnAttr(Attribute::NoRedZone);
633   User->setVisibility(GlobalValue::HiddenVisibility);
634   if (Triple(M->getTargetTriple()).supportsCOMDAT())
635     User->setComdat(M->getOrInsertComdat(User->getName()));
636 
637   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
638   auto *Load = IRB.CreateLoad(Var);
639   IRB.CreateRet(Load);
640 
641   // Mark the user variable as used so that it isn't stripped out.
642   UsedVars.push_back(User);
643 }
644 
645 void InstrProfiling::emitUses() {
646   if (!UsedVars.empty())
647     appendToUsed(*M, UsedVars);
648 }
649 
650 void InstrProfiling::emitInitialization() {
651   StringRef InstrProfileOutput = Options.InstrProfileOutput;
652 
653   if (!InstrProfileOutput.empty()) {
654     // Create variable for profile name.
655     Constant *ProfileNameConst =
656         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
657     GlobalVariable *ProfileNameVar = new GlobalVariable(
658         *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
659         ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
660     if (TT.supportsCOMDAT()) {
661       ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
662       ProfileNameVar->setComdat(M->getOrInsertComdat(
663           StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
664     }
665   }
666 
667   Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
668   if (!RegisterF)
669     return;
670 
671   // Create the initialization function.
672   auto *VoidTy = Type::getVoidTy(M->getContext());
673   auto *F = Function::Create(FunctionType::get(VoidTy, false),
674                              GlobalValue::InternalLinkage,
675                              getInstrProfInitFuncName(), M);
676   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
677   F->addFnAttr(Attribute::NoInline);
678   if (Options.NoRedZone)
679     F->addFnAttr(Attribute::NoRedZone);
680 
681   // Add the basic block and the necessary calls.
682   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
683   if (RegisterF)
684     IRB.CreateCall(RegisterF, {});
685   IRB.CreateRetVoid();
686 
687   appendToGlobalCtors(*M, F, 0);
688 }
689