1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11 // It also builds the data structures and initialization code needed for
12 // updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/Transforms/InstrProfiling.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/IR/Attributes.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/Constant.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/GlobalValue.h"
30 #include "llvm/IR/GlobalVariable.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/IntrinsicInst.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Pass.h"
38 #include "llvm/ProfileData/InstrProf.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Error.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Transforms/Utils/ModuleUtils.h"
44 #include <algorithm>
45 #include <cassert>
46 #include <cstddef>
47 #include <cstdint>
48 #include <string>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "instrprof"
53 
54 // The start and end values of precise value profile range for memory
55 // intrinsic sizes
56 cl::opt<std::string> MemOPSizeRange(
57     "memop-size-range",
58     cl::desc("Set the range of size in memory intrinsic calls to be profiled "
59              "precisely, in a format of <start_val>:<end_val>"),
60     cl::init(""));
61 
62 // The value that considered to be large value in  memory intrinsic.
63 cl::opt<unsigned> MemOPSizeLarge(
64     "memop-size-large",
65     cl::desc("Set large value thresthold in memory intrinsic size profiling. "
66              "Value of 0 disables the large value profiling."),
67     cl::init(8192));
68 
69 namespace {
70 
71 cl::opt<bool> DoNameCompression("enable-name-compression",
72                                 cl::desc("Enable name string compression"),
73                                 cl::init(true));
74 
75 cl::opt<bool> DoHashBasedCounterSplit(
76     "hash-based-counter-split",
77     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
78     cl::init(true));
79 
80 cl::opt<bool> ValueProfileStaticAlloc(
81     "vp-static-alloc",
82     cl::desc("Do static counter allocation for value profiler"),
83     cl::init(true));
84 
85 cl::opt<double> NumCountersPerValueSite(
86     "vp-counters-per-site",
87     cl::desc("The average number of profile counters allocated "
88              "per value profiling site."),
89     // This is set to a very small value because in real programs, only
90     // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91     // For those sites with non-zero profile, the average number of targets
92     // is usually smaller than 2.
93     cl::init(1.0));
94 
95 class InstrProfilingLegacyPass : public ModulePass {
96   InstrProfiling InstrProf;
97 
98 public:
99   static char ID;
100 
101   InstrProfilingLegacyPass() : ModulePass(ID) {}
102   InstrProfilingLegacyPass(const InstrProfOptions &Options)
103       : ModulePass(ID), InstrProf(Options) {}
104 
105   StringRef getPassName() const override {
106     return "Frontend instrumentation-based coverage lowering";
107   }
108 
109   bool runOnModule(Module &M) override {
110     return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
111   }
112 
113   void getAnalysisUsage(AnalysisUsage &AU) const override {
114     AU.setPreservesCFG();
115     AU.addRequired<TargetLibraryInfoWrapperPass>();
116   }
117 };
118 
119 } // end anonymous namespace
120 
121 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
122   auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
123   if (!run(M, TLI))
124     return PreservedAnalyses::all();
125 
126   return PreservedAnalyses::none();
127 }
128 
129 char InstrProfilingLegacyPass::ID = 0;
130 INITIALIZE_PASS_BEGIN(
131     InstrProfilingLegacyPass, "instrprof",
132     "Frontend instrumentation-based coverage lowering.", false, false)
133 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
134 INITIALIZE_PASS_END(
135     InstrProfilingLegacyPass, "instrprof",
136     "Frontend instrumentation-based coverage lowering.", false, false)
137 
138 ModulePass *
139 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
140   return new InstrProfilingLegacyPass(Options);
141 }
142 
143 bool InstrProfiling::isMachO() const {
144   return Triple(M->getTargetTriple()).isOSBinFormatMachO();
145 }
146 
147 /// Get the section name for the counter variables.
148 StringRef InstrProfiling::getCountersSection() const {
149   return getInstrProfCountersSectionName(isMachO());
150 }
151 
152 /// Get the section name for the name variables.
153 StringRef InstrProfiling::getNameSection() const {
154   return getInstrProfNameSectionName(isMachO());
155 }
156 
157 /// Get the section name for the profile data variables.
158 StringRef InstrProfiling::getDataSection() const {
159   return getInstrProfDataSectionName(isMachO());
160 }
161 
162 /// Get the section name for the coverage mapping data.
163 StringRef InstrProfiling::getCoverageSection() const {
164   return getInstrProfCoverageSectionName(isMachO());
165 }
166 
167 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
168   InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
169   if (Inc)
170     return Inc;
171   return dyn_cast<InstrProfIncrementInst>(Instr);
172 }
173 
174 bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
175   bool MadeChange = false;
176 
177   this->M = &M;
178   this->TLI = &TLI;
179   NamesVar = nullptr;
180   NamesSize = 0;
181   ProfileDataMap.clear();
182   UsedVars.clear();
183   getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
184                               MemOPSizeRangeLast);
185 
186   // We did not know how many value sites there would be inside
187   // the instrumented function. This is counting the number of instrumented
188   // target value sites to enter it as field in the profile data variable.
189   for (Function &F : M) {
190     InstrProfIncrementInst *FirstProfIncInst = nullptr;
191     for (BasicBlock &BB : F)
192       for (auto I = BB.begin(), E = BB.end(); I != E; I++)
193         if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
194           computeNumValueSiteCounts(Ind);
195         else if (FirstProfIncInst == nullptr)
196           FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
197 
198     // Value profiling intrinsic lowering requires per-function profile data
199     // variable to be created first.
200     if (FirstProfIncInst != nullptr)
201       static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
202   }
203 
204   for (Function &F : M)
205     for (BasicBlock &BB : F)
206       for (auto I = BB.begin(), E = BB.end(); I != E;) {
207         auto Instr = I++;
208         InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
209         if (Inc) {
210           lowerIncrement(Inc);
211           MadeChange = true;
212         } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
213           lowerValueProfileInst(Ind);
214           MadeChange = true;
215         }
216       }
217 
218   if (GlobalVariable *CoverageNamesVar =
219           M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
220     lowerCoverageData(CoverageNamesVar);
221     MadeChange = true;
222   }
223 
224   if (!MadeChange)
225     return false;
226 
227   emitVNodes();
228   emitNameData();
229   emitRegistration();
230   emitRuntimeHook();
231   emitUses();
232   emitInitialization();
233   return true;
234 }
235 
236 static Constant *getOrInsertValueProfilingCall(Module &M,
237                                                const TargetLibraryInfo &TLI,
238                                                bool IsRange = false) {
239   LLVMContext &Ctx = M.getContext();
240   auto *ReturnTy = Type::getVoidTy(M.getContext());
241 
242   Constant *Res;
243   if (!IsRange) {
244     Type *ParamTypes[] = {
245 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
246 #include "llvm/ProfileData/InstrProfData.inc"
247     };
248     auto *ValueProfilingCallTy =
249         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
250     Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
251                                 ValueProfilingCallTy);
252   } else {
253     Type *RangeParamTypes[] = {
254 #define VALUE_RANGE_PROF 1
255 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
256 #include "llvm/ProfileData/InstrProfData.inc"
257 #undef VALUE_RANGE_PROF
258     };
259     auto *ValueRangeProfilingCallTy =
260         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
261     Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
262                                 ValueRangeProfilingCallTy);
263   }
264 
265   if (Function *FunRes = dyn_cast<Function>(Res)) {
266     if (auto AK = TLI.getExtAttrForI32Param(false))
267       FunRes->addAttribute(3, AK);
268   }
269   return Res;
270 }
271 
272 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
273   GlobalVariable *Name = Ind->getName();
274   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
275   uint64_t Index = Ind->getIndex()->getZExtValue();
276   auto It = ProfileDataMap.find(Name);
277   if (It == ProfileDataMap.end()) {
278     PerFunctionProfileData PD;
279     PD.NumValueSites[ValueKind] = Index + 1;
280     ProfileDataMap[Name] = PD;
281   } else if (It->second.NumValueSites[ValueKind] <= Index)
282     It->second.NumValueSites[ValueKind] = Index + 1;
283 }
284 
285 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
286   GlobalVariable *Name = Ind->getName();
287   auto It = ProfileDataMap.find(Name);
288   assert(It != ProfileDataMap.end() && It->second.DataVar &&
289          "value profiling detected in function with no counter incerement");
290 
291   GlobalVariable *DataVar = It->second.DataVar;
292   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
293   uint64_t Index = Ind->getIndex()->getZExtValue();
294   for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
295     Index += It->second.NumValueSites[Kind];
296 
297   IRBuilder<> Builder(Ind);
298   bool IsRange = (Ind->getValueKind()->getZExtValue() ==
299                   llvm::InstrProfValueKind::IPVK_MemOPSize);
300   CallInst *Call = nullptr;
301   if (!IsRange) {
302     Value *Args[3] = {Ind->getTargetValue(),
303                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
304                       Builder.getInt32(Index)};
305     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
306   } else {
307     Value *Args[6] = {
308         Ind->getTargetValue(),
309         Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
310         Builder.getInt32(Index),
311         Builder.getInt64(MemOPSizeRangeStart),
312         Builder.getInt64(MemOPSizeRangeLast),
313         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
314     Call =
315         Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
316   }
317   if (auto AK = TLI->getExtAttrForI32Param(false))
318     Call->addAttribute(3, AK);
319   Ind->replaceAllUsesWith(Call);
320   Ind->eraseFromParent();
321 }
322 
323 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
324   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
325 
326   IRBuilder<> Builder(Inc);
327   uint64_t Index = Inc->getIndex()->getZExtValue();
328   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
329   Value *Count = Builder.CreateLoad(Addr, "pgocount");
330   Count = Builder.CreateAdd(Count, Inc->getStep());
331   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
332   Inc->eraseFromParent();
333 }
334 
335 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
336   ConstantArray *Names =
337       cast<ConstantArray>(CoverageNamesVar->getInitializer());
338   for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
339     Constant *NC = Names->getOperand(I);
340     Value *V = NC->stripPointerCasts();
341     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
342     GlobalVariable *Name = cast<GlobalVariable>(V);
343 
344     Name->setLinkage(GlobalValue::PrivateLinkage);
345     ReferencedNames.push_back(Name);
346     NC->dropAllReferences();
347   }
348   CoverageNamesVar->eraseFromParent();
349 }
350 
351 /// Get the name of a profiling variable for a particular function.
352 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
353   StringRef NamePrefix = getInstrProfNameVarPrefix();
354   StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
355   Function *F = Inc->getParent()->getParent();
356   Module *M = F->getParent();
357   if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
358       !canRenameComdatFunc(*F))
359     return (Prefix + Name).str();
360   uint64_t FuncHash = Inc->getHash()->getZExtValue();
361   SmallVector<char, 24> HashPostfix;
362   if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
363     return (Prefix + Name).str();
364   return (Prefix + Name + "." + Twine(FuncHash)).str();
365 }
366 
367 static inline bool shouldRecordFunctionAddr(Function *F) {
368   // Check the linkage
369   if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
370       !F->hasAvailableExternallyLinkage())
371     return true;
372   // Prohibit function address recording if the function is both internal and
373   // COMDAT. This avoids the profile data variable referencing internal symbols
374   // in COMDAT.
375   if (F->hasLocalLinkage() && F->hasComdat())
376     return false;
377   // Check uses of this function for other than direct calls or invokes to it.
378   // Inline virtual functions have linkeOnceODR linkage. When a key method
379   // exists, the vtable will only be emitted in the TU where the key method
380   // is defined. In a TU where vtable is not available, the function won't
381   // be 'addresstaken'. If its address is not recorded here, the profile data
382   // with missing address may be picked by the linker leading  to missing
383   // indirect call target info.
384   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
385 }
386 
387 static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
388                                                InstrProfIncrementInst *Inc) {
389   if (!needsComdatForCounter(F, M))
390     return nullptr;
391 
392   // COFF format requires a COMDAT section to have a key symbol with the same
393   // name. The linker targeting COFF also requires that the COMDAT
394   // a section is associated to must precede the associating section. For this
395   // reason, we must choose the counter var's name as the name of the comdat.
396   StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
397                                 ? getInstrProfCountersVarPrefix()
398                                 : getInstrProfComdatPrefix());
399   return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
400 }
401 
402 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
403   // Don't do this for Darwin.  compiler-rt uses linker magic.
404   if (Triple(M.getTargetTriple()).isOSDarwin())
405     return false;
406 
407   // Use linker script magic to get data/cnts/name start/end.
408   if (Triple(M.getTargetTriple()).isOSLinux() ||
409       Triple(M.getTargetTriple()).isOSFreeBSD() ||
410       Triple(M.getTargetTriple()).isPS4CPU())
411     return false;
412 
413   return true;
414 }
415 
416 GlobalVariable *
417 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
418   GlobalVariable *NamePtr = Inc->getName();
419   auto It = ProfileDataMap.find(NamePtr);
420   PerFunctionProfileData PD;
421   if (It != ProfileDataMap.end()) {
422     if (It->second.RegionCounters)
423       return It->second.RegionCounters;
424     PD = It->second;
425   }
426 
427   // Move the name variable to the right section. Place them in a COMDAT group
428   // if the associated function is a COMDAT. This will make sure that
429   // only one copy of counters of the COMDAT function will be emitted after
430   // linking.
431   Function *Fn = Inc->getParent()->getParent();
432   Comdat *ProfileVarsComdat = nullptr;
433   ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
434 
435   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
436   LLVMContext &Ctx = M->getContext();
437   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
438 
439   // Create the counters variable.
440   auto *CounterPtr =
441       new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
442                          Constant::getNullValue(CounterTy),
443                          getVarName(Inc, getInstrProfCountersVarPrefix()));
444   CounterPtr->setVisibility(NamePtr->getVisibility());
445   CounterPtr->setSection(getCountersSection());
446   CounterPtr->setAlignment(8);
447   CounterPtr->setComdat(ProfileVarsComdat);
448 
449   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
450   // Allocate statically the array of pointers to value profile nodes for
451   // the current function.
452   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
453   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
454     uint64_t NS = 0;
455     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
456       NS += PD.NumValueSites[Kind];
457     if (NS) {
458       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
459 
460       auto *ValuesVar =
461           new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
462                              Constant::getNullValue(ValuesTy),
463                              getVarName(Inc, getInstrProfValuesVarPrefix()));
464       ValuesVar->setVisibility(NamePtr->getVisibility());
465       ValuesVar->setSection(getInstrProfValuesSectionName(isMachO()));
466       ValuesVar->setAlignment(8);
467       ValuesVar->setComdat(ProfileVarsComdat);
468       ValuesPtrExpr =
469           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
470     }
471   }
472 
473   // Create data variable.
474   auto *Int16Ty = Type::getInt16Ty(Ctx);
475   auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
476   Type *DataTypes[] = {
477 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
478 #include "llvm/ProfileData/InstrProfData.inc"
479   };
480   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
481 
482   Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
483                                ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
484                                : ConstantPointerNull::get(Int8PtrTy);
485 
486   Constant *Int16ArrayVals[IPVK_Last + 1];
487   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
488     Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
489 
490   Constant *DataVals[] = {
491 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
492 #include "llvm/ProfileData/InstrProfData.inc"
493   };
494   auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
495                                   ConstantStruct::get(DataTy, DataVals),
496                                   getVarName(Inc, getInstrProfDataVarPrefix()));
497   Data->setVisibility(NamePtr->getVisibility());
498   Data->setSection(getDataSection());
499   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
500   Data->setComdat(ProfileVarsComdat);
501 
502   PD.RegionCounters = CounterPtr;
503   PD.DataVar = Data;
504   ProfileDataMap[NamePtr] = PD;
505 
506   // Mark the data variable as used so that it isn't stripped out.
507   UsedVars.push_back(Data);
508   // Now that the linkage set by the FE has been passed to the data and counter
509   // variables, reset Name variable's linkage and visibility to private so that
510   // it can be removed later by the compiler.
511   NamePtr->setLinkage(GlobalValue::PrivateLinkage);
512   // Collect the referenced names to be used by emitNameData.
513   ReferencedNames.push_back(NamePtr);
514 
515   return CounterPtr;
516 }
517 
518 void InstrProfiling::emitVNodes() {
519   if (!ValueProfileStaticAlloc)
520     return;
521 
522   // For now only support this on platforms that do
523   // not require runtime registration to discover
524   // named section start/end.
525   if (needsRuntimeRegistrationOfSectionRange(*M))
526     return;
527 
528   size_t TotalNS = 0;
529   for (auto &PD : ProfileDataMap) {
530     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
531       TotalNS += PD.second.NumValueSites[Kind];
532   }
533 
534   if (!TotalNS)
535     return;
536 
537   uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
538 // Heuristic for small programs with very few total value sites.
539 // The default value of vp-counters-per-site is chosen based on
540 // the observation that large apps usually have a low percentage
541 // of value sites that actually have any profile data, and thus
542 // the average number of counters per site is low. For small
543 // apps with very few sites, this may not be true. Bump up the
544 // number of counters in this case.
545 #define INSTR_PROF_MIN_VAL_COUNTS 10
546   if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
547     NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
548 
549   auto &Ctx = M->getContext();
550   Type *VNodeTypes[] = {
551 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
552 #include "llvm/ProfileData/InstrProfData.inc"
553   };
554   auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
555 
556   ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
557   auto *VNodesVar = new GlobalVariable(
558       *M, VNodesTy, false, GlobalValue::PrivateLinkage,
559       Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
560   VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO()));
561   UsedVars.push_back(VNodesVar);
562 }
563 
564 void InstrProfiling::emitNameData() {
565   std::string UncompressedData;
566 
567   if (ReferencedNames.empty())
568     return;
569 
570   std::string CompressedNameStr;
571   if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
572                                           DoNameCompression)) {
573     report_fatal_error(toString(std::move(E)), false);
574   }
575 
576   auto &Ctx = M->getContext();
577   auto *NamesVal = ConstantDataArray::getString(
578       Ctx, StringRef(CompressedNameStr), false);
579   NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
580                                 GlobalValue::PrivateLinkage, NamesVal,
581                                 getInstrProfNamesVarName());
582   NamesSize = CompressedNameStr.size();
583   NamesVar->setSection(getNameSection());
584   UsedVars.push_back(NamesVar);
585 
586   for (auto *NamePtr : ReferencedNames)
587     NamePtr->eraseFromParent();
588 }
589 
590 void InstrProfiling::emitRegistration() {
591   if (!needsRuntimeRegistrationOfSectionRange(*M))
592     return;
593 
594   // Construct the function.
595   auto *VoidTy = Type::getVoidTy(M->getContext());
596   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
597   auto *Int64Ty = Type::getInt64Ty(M->getContext());
598   auto *RegisterFTy = FunctionType::get(VoidTy, false);
599   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
600                                      getInstrProfRegFuncsName(), M);
601   RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
602   if (Options.NoRedZone)
603     RegisterF->addFnAttr(Attribute::NoRedZone);
604 
605   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
606   auto *RuntimeRegisterF =
607       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
608                        getInstrProfRegFuncName(), M);
609 
610   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
611   for (Value *Data : UsedVars)
612     if (Data != NamesVar)
613       IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
614 
615   if (NamesVar) {
616     Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
617     auto *NamesRegisterTy =
618         FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
619     auto *NamesRegisterF =
620         Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
621                          getInstrProfNamesRegFuncName(), M);
622     IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
623                                     IRB.getInt64(NamesSize)});
624   }
625 
626   IRB.CreateRetVoid();
627 }
628 
629 void InstrProfiling::emitRuntimeHook() {
630   // We expect the linker to be invoked with -u<hook_var> flag for linux,
631   // for which case there is no need to emit the user function.
632   if (Triple(M->getTargetTriple()).isOSLinux())
633     return;
634 
635   // If the module's provided its own runtime, we don't need to do anything.
636   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
637     return;
638 
639   // Declare an external variable that will pull in the runtime initialization.
640   auto *Int32Ty = Type::getInt32Ty(M->getContext());
641   auto *Var =
642       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
643                          nullptr, getInstrProfRuntimeHookVarName());
644 
645   // Make a function that uses it.
646   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
647                                 GlobalValue::LinkOnceODRLinkage,
648                                 getInstrProfRuntimeHookVarUseFuncName(), M);
649   User->addFnAttr(Attribute::NoInline);
650   if (Options.NoRedZone)
651     User->addFnAttr(Attribute::NoRedZone);
652   User->setVisibility(GlobalValue::HiddenVisibility);
653   if (Triple(M->getTargetTriple()).supportsCOMDAT())
654     User->setComdat(M->getOrInsertComdat(User->getName()));
655 
656   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
657   auto *Load = IRB.CreateLoad(Var);
658   IRB.CreateRet(Load);
659 
660   // Mark the user variable as used so that it isn't stripped out.
661   UsedVars.push_back(User);
662 }
663 
664 void InstrProfiling::emitUses() {
665   if (!UsedVars.empty())
666     appendToUsed(*M, UsedVars);
667 }
668 
669 void InstrProfiling::emitInitialization() {
670   StringRef InstrProfileOutput = Options.InstrProfileOutput;
671 
672   if (!InstrProfileOutput.empty()) {
673     // Create variable for profile name.
674     Constant *ProfileNameConst =
675         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
676     GlobalVariable *ProfileNameVar = new GlobalVariable(
677         *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
678         ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
679     Triple TT(M->getTargetTriple());
680     if (TT.supportsCOMDAT()) {
681       ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
682       ProfileNameVar->setComdat(M->getOrInsertComdat(
683           StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
684     }
685   }
686 
687   Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
688   if (!RegisterF)
689     return;
690 
691   // Create the initialization function.
692   auto *VoidTy = Type::getVoidTy(M->getContext());
693   auto *F = Function::Create(FunctionType::get(VoidTy, false),
694                              GlobalValue::InternalLinkage,
695                              getInstrProfInitFuncName(), M);
696   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
697   F->addFnAttr(Attribute::NoInline);
698   if (Options.NoRedZone)
699     F->addFnAttr(Attribute::NoRedZone);
700 
701   // Add the basic block and the necessary calls.
702   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
703   if (RegisterF)
704     IRB.CreateCall(RegisterF, {});
705   IRB.CreateRetVoid();
706 
707   appendToGlobalCtors(*M, F, 0);
708 }
709