1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11 // It also builds the data structures and initialization code needed for
12 // updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/Transforms/InstrProfiling.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/IR/Attributes.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/Constant.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/GlobalValue.h"
30 #include "llvm/IR/GlobalVariable.h"
31 #include "llvm/IR/IRBuilder.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Pass.h"
38 #include "llvm/ProfileData/InstrProf.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Error.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Transforms/Utils/ModuleUtils.h"
44 #include <algorithm>
45 #include <cassert>
46 #include <cstddef>
47 #include <cstdint>
48 #include <string>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "instrprof"
53 
54 // The start and end values of precise value profile range for memory
55 // intrinsic sizes
56 cl::opt<std::string> MemOPSizeRange(
57     "memop-size-range",
58     cl::desc("Set the range of size in memory intrinsic calls to be profiled "
59              "precisely, in a format of <start_val>:<end_val>"),
60     cl::init(""));
61 
62 // The value that considered to be large value in  memory intrinsic.
63 cl::opt<unsigned> MemOPSizeLarge(
64     "memop-size-large",
65     cl::desc("Set large value thresthold in memory intrinsic size profiling. "
66              "Value of 0 disables the large value profiling."),
67     cl::init(8192));
68 
69 namespace {
70 
71 cl::opt<bool> DoNameCompression("enable-name-compression",
72                                 cl::desc("Enable name string compression"),
73                                 cl::init(true));
74 
75 cl::opt<bool> DoHashBasedCounterSplit(
76     "hash-based-counter-split",
77     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
78     cl::init(true));
79 
80 cl::opt<bool> ValueProfileStaticAlloc(
81     "vp-static-alloc",
82     cl::desc("Do static counter allocation for value profiler"),
83     cl::init(true));
84 
85 cl::opt<double> NumCountersPerValueSite(
86     "vp-counters-per-site",
87     cl::desc("The average number of profile counters allocated "
88              "per value profiling site."),
89     // This is set to a very small value because in real programs, only
90     // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91     // For those sites with non-zero profile, the average number of targets
92     // is usually smaller than 2.
93     cl::init(1.0));
94 
95 class InstrProfilingLegacyPass : public ModulePass {
96   InstrProfiling InstrProf;
97 
98 public:
99   static char ID;
100 
101   InstrProfilingLegacyPass() : ModulePass(ID) {}
102   InstrProfilingLegacyPass(const InstrProfOptions &Options)
103       : ModulePass(ID), InstrProf(Options) {}
104 
105   StringRef getPassName() const override {
106     return "Frontend instrumentation-based coverage lowering";
107   }
108 
109   bool runOnModule(Module &M) override {
110     return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
111   }
112 
113   void getAnalysisUsage(AnalysisUsage &AU) const override {
114     AU.setPreservesCFG();
115     AU.addRequired<TargetLibraryInfoWrapperPass>();
116   }
117 };
118 
119 } // end anonymous namespace
120 
121 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
122   auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
123   if (!run(M, TLI))
124     return PreservedAnalyses::all();
125 
126   return PreservedAnalyses::none();
127 }
128 
129 char InstrProfilingLegacyPass::ID = 0;
130 INITIALIZE_PASS_BEGIN(
131     InstrProfilingLegacyPass, "instrprof",
132     "Frontend instrumentation-based coverage lowering.", false, false)
133 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
134 INITIALIZE_PASS_END(
135     InstrProfilingLegacyPass, "instrprof",
136     "Frontend instrumentation-based coverage lowering.", false, false)
137 
138 ModulePass *
139 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
140   return new InstrProfilingLegacyPass(Options);
141 }
142 
143 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
144   InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
145   if (Inc)
146     return Inc;
147   return dyn_cast<InstrProfIncrementInst>(Instr);
148 }
149 
150 bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
151   bool MadeChange = false;
152 
153   this->M = &M;
154   this->TLI = &TLI;
155   NamesVar = nullptr;
156   NamesSize = 0;
157   ProfileDataMap.clear();
158   UsedVars.clear();
159   getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
160                               MemOPSizeRangeLast);
161   TT = Triple(M.getTargetTriple());
162 
163   // We did not know how many value sites there would be inside
164   // the instrumented function. This is counting the number of instrumented
165   // target value sites to enter it as field in the profile data variable.
166   for (Function &F : M) {
167     InstrProfIncrementInst *FirstProfIncInst = nullptr;
168     for (BasicBlock &BB : F)
169       for (auto I = BB.begin(), E = BB.end(); I != E; I++)
170         if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
171           computeNumValueSiteCounts(Ind);
172         else if (FirstProfIncInst == nullptr)
173           FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
174 
175     // Value profiling intrinsic lowering requires per-function profile data
176     // variable to be created first.
177     if (FirstProfIncInst != nullptr)
178       static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
179   }
180 
181   for (Function &F : M)
182     for (BasicBlock &BB : F)
183       for (auto I = BB.begin(), E = BB.end(); I != E;) {
184         auto Instr = I++;
185         InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
186         if (Inc) {
187           lowerIncrement(Inc);
188           MadeChange = true;
189         } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
190           lowerValueProfileInst(Ind);
191           MadeChange = true;
192         }
193       }
194 
195   if (GlobalVariable *CoverageNamesVar =
196           M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
197     lowerCoverageData(CoverageNamesVar);
198     MadeChange = true;
199   }
200 
201   if (!MadeChange)
202     return false;
203 
204   emitVNodes();
205   emitNameData();
206   emitRegistration();
207   emitRuntimeHook();
208   emitUses();
209   emitInitialization();
210   return true;
211 }
212 
213 static Constant *getOrInsertValueProfilingCall(Module &M,
214                                                const TargetLibraryInfo &TLI,
215                                                bool IsRange = false) {
216   LLVMContext &Ctx = M.getContext();
217   auto *ReturnTy = Type::getVoidTy(M.getContext());
218 
219   Constant *Res;
220   if (!IsRange) {
221     Type *ParamTypes[] = {
222 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
223 #include "llvm/ProfileData/InstrProfData.inc"
224     };
225     auto *ValueProfilingCallTy =
226         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
227     Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
228                                 ValueProfilingCallTy);
229   } else {
230     Type *RangeParamTypes[] = {
231 #define VALUE_RANGE_PROF 1
232 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
233 #include "llvm/ProfileData/InstrProfData.inc"
234 #undef VALUE_RANGE_PROF
235     };
236     auto *ValueRangeProfilingCallTy =
237         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
238     Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
239                                 ValueRangeProfilingCallTy);
240   }
241 
242   if (Function *FunRes = dyn_cast<Function>(Res)) {
243     if (auto AK = TLI.getExtAttrForI32Param(false))
244       FunRes->addParamAttr(2, AK);
245   }
246   return Res;
247 }
248 
249 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
250   GlobalVariable *Name = Ind->getName();
251   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
252   uint64_t Index = Ind->getIndex()->getZExtValue();
253   auto It = ProfileDataMap.find(Name);
254   if (It == ProfileDataMap.end()) {
255     PerFunctionProfileData PD;
256     PD.NumValueSites[ValueKind] = Index + 1;
257     ProfileDataMap[Name] = PD;
258   } else if (It->second.NumValueSites[ValueKind] <= Index)
259     It->second.NumValueSites[ValueKind] = Index + 1;
260 }
261 
262 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
263   GlobalVariable *Name = Ind->getName();
264   auto It = ProfileDataMap.find(Name);
265   assert(It != ProfileDataMap.end() && It->second.DataVar &&
266          "value profiling detected in function with no counter incerement");
267 
268   GlobalVariable *DataVar = It->second.DataVar;
269   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
270   uint64_t Index = Ind->getIndex()->getZExtValue();
271   for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
272     Index += It->second.NumValueSites[Kind];
273 
274   IRBuilder<> Builder(Ind);
275   bool IsRange = (Ind->getValueKind()->getZExtValue() ==
276                   llvm::InstrProfValueKind::IPVK_MemOPSize);
277   CallInst *Call = nullptr;
278   if (!IsRange) {
279     Value *Args[3] = {Ind->getTargetValue(),
280                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
281                       Builder.getInt32(Index)};
282     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
283   } else {
284     Value *Args[6] = {
285         Ind->getTargetValue(),
286         Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
287         Builder.getInt32(Index),
288         Builder.getInt64(MemOPSizeRangeStart),
289         Builder.getInt64(MemOPSizeRangeLast),
290         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
291     Call =
292         Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
293   }
294   if (auto AK = TLI->getExtAttrForI32Param(false))
295     Call->addParamAttr(2, AK);
296   Ind->replaceAllUsesWith(Call);
297   Ind->eraseFromParent();
298 }
299 
300 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
301   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
302 
303   IRBuilder<> Builder(Inc);
304   uint64_t Index = Inc->getIndex()->getZExtValue();
305   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
306   Value *Count = Builder.CreateLoad(Addr, "pgocount");
307   Count = Builder.CreateAdd(Count, Inc->getStep());
308   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
309   Inc->eraseFromParent();
310 }
311 
312 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
313   ConstantArray *Names =
314       cast<ConstantArray>(CoverageNamesVar->getInitializer());
315   for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
316     Constant *NC = Names->getOperand(I);
317     Value *V = NC->stripPointerCasts();
318     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
319     GlobalVariable *Name = cast<GlobalVariable>(V);
320 
321     Name->setLinkage(GlobalValue::PrivateLinkage);
322     ReferencedNames.push_back(Name);
323     NC->dropAllReferences();
324   }
325   CoverageNamesVar->eraseFromParent();
326 }
327 
328 /// Get the name of a profiling variable for a particular function.
329 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
330   StringRef NamePrefix = getInstrProfNameVarPrefix();
331   StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
332   Function *F = Inc->getParent()->getParent();
333   Module *M = F->getParent();
334   if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
335       !canRenameComdatFunc(*F))
336     return (Prefix + Name).str();
337   uint64_t FuncHash = Inc->getHash()->getZExtValue();
338   SmallVector<char, 24> HashPostfix;
339   if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
340     return (Prefix + Name).str();
341   return (Prefix + Name + "." + Twine(FuncHash)).str();
342 }
343 
344 static inline bool shouldRecordFunctionAddr(Function *F) {
345   // Check the linkage
346   bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
347   if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
348       !HasAvailableExternallyLinkage)
349     return true;
350 
351   // A function marked 'alwaysinline' with available_externally linkage can't
352   // have its address taken. Doing so would create an undefined external ref to
353   // the function, which would fail to link.
354   if (HasAvailableExternallyLinkage &&
355       F->hasFnAttribute(Attribute::AlwaysInline))
356     return false;
357 
358   // Prohibit function address recording if the function is both internal and
359   // COMDAT. This avoids the profile data variable referencing internal symbols
360   // in COMDAT.
361   if (F->hasLocalLinkage() && F->hasComdat())
362     return false;
363 
364   // Check uses of this function for other than direct calls or invokes to it.
365   // Inline virtual functions have linkeOnceODR linkage. When a key method
366   // exists, the vtable will only be emitted in the TU where the key method
367   // is defined. In a TU where vtable is not available, the function won't
368   // be 'addresstaken'. If its address is not recorded here, the profile data
369   // with missing address may be picked by the linker leading  to missing
370   // indirect call target info.
371   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
372 }
373 
374 static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
375                                                InstrProfIncrementInst *Inc) {
376   if (!needsComdatForCounter(F, M))
377     return nullptr;
378 
379   // COFF format requires a COMDAT section to have a key symbol with the same
380   // name. The linker targeting COFF also requires that the COMDAT
381   // a section is associated to must precede the associating section. For this
382   // reason, we must choose the counter var's name as the name of the comdat.
383   StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
384                                 ? getInstrProfCountersVarPrefix()
385                                 : getInstrProfComdatPrefix());
386   return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
387 }
388 
389 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
390   // Don't do this for Darwin.  compiler-rt uses linker magic.
391   if (Triple(M.getTargetTriple()).isOSDarwin())
392     return false;
393 
394   // Use linker script magic to get data/cnts/name start/end.
395   if (Triple(M.getTargetTriple()).isOSLinux() ||
396       Triple(M.getTargetTriple()).isOSFreeBSD() ||
397       Triple(M.getTargetTriple()).isPS4CPU())
398     return false;
399 
400   return true;
401 }
402 
403 GlobalVariable *
404 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
405   GlobalVariable *NamePtr = Inc->getName();
406   auto It = ProfileDataMap.find(NamePtr);
407   PerFunctionProfileData PD;
408   if (It != ProfileDataMap.end()) {
409     if (It->second.RegionCounters)
410       return It->second.RegionCounters;
411     PD = It->second;
412   }
413 
414   // Move the name variable to the right section. Place them in a COMDAT group
415   // if the associated function is a COMDAT. This will make sure that
416   // only one copy of counters of the COMDAT function will be emitted after
417   // linking.
418   Function *Fn = Inc->getParent()->getParent();
419   Comdat *ProfileVarsComdat = nullptr;
420   ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
421 
422   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
423   LLVMContext &Ctx = M->getContext();
424   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
425 
426   // Create the counters variable.
427   auto *CounterPtr =
428       new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
429                          Constant::getNullValue(CounterTy),
430                          getVarName(Inc, getInstrProfCountersVarPrefix()));
431   CounterPtr->setVisibility(NamePtr->getVisibility());
432   CounterPtr->setSection(
433       getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
434   CounterPtr->setAlignment(8);
435   CounterPtr->setComdat(ProfileVarsComdat);
436 
437   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
438   // Allocate statically the array of pointers to value profile nodes for
439   // the current function.
440   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
441   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
442     uint64_t NS = 0;
443     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
444       NS += PD.NumValueSites[Kind];
445     if (NS) {
446       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
447 
448       auto *ValuesVar =
449           new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
450                              Constant::getNullValue(ValuesTy),
451                              getVarName(Inc, getInstrProfValuesVarPrefix()));
452       ValuesVar->setVisibility(NamePtr->getVisibility());
453       ValuesVar->setSection(
454           getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
455       ValuesVar->setAlignment(8);
456       ValuesVar->setComdat(ProfileVarsComdat);
457       ValuesPtrExpr =
458           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
459     }
460   }
461 
462   // Create data variable.
463   auto *Int16Ty = Type::getInt16Ty(Ctx);
464   auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
465   Type *DataTypes[] = {
466 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
467 #include "llvm/ProfileData/InstrProfData.inc"
468   };
469   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
470 
471   Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
472                                ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
473                                : ConstantPointerNull::get(Int8PtrTy);
474 
475   Constant *Int16ArrayVals[IPVK_Last + 1];
476   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
477     Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
478 
479   Constant *DataVals[] = {
480 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
481 #include "llvm/ProfileData/InstrProfData.inc"
482   };
483   auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
484                                   ConstantStruct::get(DataTy, DataVals),
485                                   getVarName(Inc, getInstrProfDataVarPrefix()));
486   Data->setVisibility(NamePtr->getVisibility());
487   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
488   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
489   Data->setComdat(ProfileVarsComdat);
490 
491   PD.RegionCounters = CounterPtr;
492   PD.DataVar = Data;
493   ProfileDataMap[NamePtr] = PD;
494 
495   // Mark the data variable as used so that it isn't stripped out.
496   UsedVars.push_back(Data);
497   // Now that the linkage set by the FE has been passed to the data and counter
498   // variables, reset Name variable's linkage and visibility to private so that
499   // it can be removed later by the compiler.
500   NamePtr->setLinkage(GlobalValue::PrivateLinkage);
501   // Collect the referenced names to be used by emitNameData.
502   ReferencedNames.push_back(NamePtr);
503 
504   return CounterPtr;
505 }
506 
507 void InstrProfiling::emitVNodes() {
508   if (!ValueProfileStaticAlloc)
509     return;
510 
511   // For now only support this on platforms that do
512   // not require runtime registration to discover
513   // named section start/end.
514   if (needsRuntimeRegistrationOfSectionRange(*M))
515     return;
516 
517   size_t TotalNS = 0;
518   for (auto &PD : ProfileDataMap) {
519     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
520       TotalNS += PD.second.NumValueSites[Kind];
521   }
522 
523   if (!TotalNS)
524     return;
525 
526   uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
527 // Heuristic for small programs with very few total value sites.
528 // The default value of vp-counters-per-site is chosen based on
529 // the observation that large apps usually have a low percentage
530 // of value sites that actually have any profile data, and thus
531 // the average number of counters per site is low. For small
532 // apps with very few sites, this may not be true. Bump up the
533 // number of counters in this case.
534 #define INSTR_PROF_MIN_VAL_COUNTS 10
535   if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
536     NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
537 
538   auto &Ctx = M->getContext();
539   Type *VNodeTypes[] = {
540 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
541 #include "llvm/ProfileData/InstrProfData.inc"
542   };
543   auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
544 
545   ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
546   auto *VNodesVar = new GlobalVariable(
547       *M, VNodesTy, false, GlobalValue::PrivateLinkage,
548       Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
549   VNodesVar->setSection(
550       getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
551   UsedVars.push_back(VNodesVar);
552 }
553 
554 void InstrProfiling::emitNameData() {
555   std::string UncompressedData;
556 
557   if (ReferencedNames.empty())
558     return;
559 
560   std::string CompressedNameStr;
561   if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
562                                           DoNameCompression)) {
563     report_fatal_error(toString(std::move(E)), false);
564   }
565 
566   auto &Ctx = M->getContext();
567   auto *NamesVal = ConstantDataArray::getString(
568       Ctx, StringRef(CompressedNameStr), false);
569   NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
570                                 GlobalValue::PrivateLinkage, NamesVal,
571                                 getInstrProfNamesVarName());
572   NamesSize = CompressedNameStr.size();
573   NamesVar->setSection(
574       getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
575   UsedVars.push_back(NamesVar);
576 
577   for (auto *NamePtr : ReferencedNames)
578     NamePtr->eraseFromParent();
579 }
580 
581 void InstrProfiling::emitRegistration() {
582   if (!needsRuntimeRegistrationOfSectionRange(*M))
583     return;
584 
585   // Construct the function.
586   auto *VoidTy = Type::getVoidTy(M->getContext());
587   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
588   auto *Int64Ty = Type::getInt64Ty(M->getContext());
589   auto *RegisterFTy = FunctionType::get(VoidTy, false);
590   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
591                                      getInstrProfRegFuncsName(), M);
592   RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
593   if (Options.NoRedZone)
594     RegisterF->addFnAttr(Attribute::NoRedZone);
595 
596   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
597   auto *RuntimeRegisterF =
598       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
599                        getInstrProfRegFuncName(), M);
600 
601   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
602   for (Value *Data : UsedVars)
603     if (Data != NamesVar)
604       IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
605 
606   if (NamesVar) {
607     Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
608     auto *NamesRegisterTy =
609         FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
610     auto *NamesRegisterF =
611         Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
612                          getInstrProfNamesRegFuncName(), M);
613     IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
614                                     IRB.getInt64(NamesSize)});
615   }
616 
617   IRB.CreateRetVoid();
618 }
619 
620 void InstrProfiling::emitRuntimeHook() {
621   // We expect the linker to be invoked with -u<hook_var> flag for linux,
622   // for which case there is no need to emit the user function.
623   if (Triple(M->getTargetTriple()).isOSLinux())
624     return;
625 
626   // If the module's provided its own runtime, we don't need to do anything.
627   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
628     return;
629 
630   // Declare an external variable that will pull in the runtime initialization.
631   auto *Int32Ty = Type::getInt32Ty(M->getContext());
632   auto *Var =
633       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
634                          nullptr, getInstrProfRuntimeHookVarName());
635 
636   // Make a function that uses it.
637   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
638                                 GlobalValue::LinkOnceODRLinkage,
639                                 getInstrProfRuntimeHookVarUseFuncName(), M);
640   User->addFnAttr(Attribute::NoInline);
641   if (Options.NoRedZone)
642     User->addFnAttr(Attribute::NoRedZone);
643   User->setVisibility(GlobalValue::HiddenVisibility);
644   if (Triple(M->getTargetTriple()).supportsCOMDAT())
645     User->setComdat(M->getOrInsertComdat(User->getName()));
646 
647   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
648   auto *Load = IRB.CreateLoad(Var);
649   IRB.CreateRet(Load);
650 
651   // Mark the user variable as used so that it isn't stripped out.
652   UsedVars.push_back(User);
653 }
654 
655 void InstrProfiling::emitUses() {
656   if (!UsedVars.empty())
657     appendToUsed(*M, UsedVars);
658 }
659 
660 void InstrProfiling::emitInitialization() {
661   StringRef InstrProfileOutput = Options.InstrProfileOutput;
662 
663   if (!InstrProfileOutput.empty()) {
664     // Create variable for profile name.
665     Constant *ProfileNameConst =
666         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
667     GlobalVariable *ProfileNameVar = new GlobalVariable(
668         *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
669         ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
670     if (TT.supportsCOMDAT()) {
671       ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
672       ProfileNameVar->setComdat(M->getOrInsertComdat(
673           StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
674     }
675   }
676 
677   Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
678   if (!RegisterF)
679     return;
680 
681   // Create the initialization function.
682   auto *VoidTy = Type::getVoidTy(M->getContext());
683   auto *F = Function::Create(FunctionType::get(VoidTy, false),
684                              GlobalValue::InternalLinkage,
685                              getInstrProfInitFuncName(), M);
686   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
687   F->addFnAttr(Attribute::NoInline);
688   if (Options.NoRedZone)
689     F->addFnAttr(Attribute::NoRedZone);
690 
691   // Add the basic block and the necessary calls.
692   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
693   if (RegisterF)
694     IRB.CreateCall(RegisterF, {});
695   IRB.CreateRetVoid();
696 
697   appendToGlobalCtors(*M, F, 0);
698 }
699