1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11 // It also builds the data structures and initialization code needed for
12 // updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/Transforms/InstrProfiling.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/IR/Attributes.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/Constant.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/GlobalValue.h"
30 #include "llvm/IR/GlobalVariable.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/IntrinsicInst.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Pass.h"
38 #include "llvm/ProfileData/InstrProf.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Error.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Transforms/Utils/ModuleUtils.h"
44 #include <algorithm>
45 #include <cassert>
46 #include <cstddef>
47 #include <cstdint>
48 #include <string>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "instrprof"
53 
54 namespace {
55 
56 cl::opt<bool> DoNameCompression("enable-name-compression",
57                                 cl::desc("Enable name string compression"),
58                                 cl::init(true));
59 
60 cl::opt<bool> DoHashBasedCounterSplit(
61     "hash-based-counter-split",
62     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
63     cl::init(true));
64 
65 cl::opt<bool> ValueProfileStaticAlloc(
66     "vp-static-alloc",
67     cl::desc("Do static counter allocation for value profiler"),
68     cl::init(true));
69 
70 cl::opt<double> NumCountersPerValueSite(
71     "vp-counters-per-site",
72     cl::desc("The average number of profile counters allocated "
73              "per value profiling site."),
74     // This is set to a very small value because in real programs, only
75     // a very small percentage of value sites have non-zero targets, e.g, 1/30.
76     // For those sites with non-zero profile, the average number of targets
77     // is usually smaller than 2.
78     cl::init(1.0));
79 
80 cl::opt<std::string> MemOPSizeRange(
81     "memop-size-range",
82     cl::desc("Set the range of size in memory intrinsic calls to be profiled "
83              "precisely, in a format of <start_val>:<end_val>"),
84     cl::init(""));
85 cl::opt<unsigned> MemOPSizeLarge(
86     "memop-size-large",
87     cl::desc("Set large value thresthold in memory intrinsic size profiling. "
88              "Value of 0 disables the large value profiling."),
89     cl::init(8192));
90 
91 class InstrProfilingLegacyPass : public ModulePass {
92   InstrProfiling InstrProf;
93 
94 public:
95   static char ID;
96 
97   InstrProfilingLegacyPass() : ModulePass(ID) {}
98   InstrProfilingLegacyPass(const InstrProfOptions &Options)
99       : ModulePass(ID), InstrProf(Options) {}
100 
101   StringRef getPassName() const override {
102     return "Frontend instrumentation-based coverage lowering";
103   }
104 
105   bool runOnModule(Module &M) override {
106     return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
107   }
108 
109   void getAnalysisUsage(AnalysisUsage &AU) const override {
110     AU.setPreservesCFG();
111     AU.addRequired<TargetLibraryInfoWrapperPass>();
112   }
113 };
114 
115 } // end anonymous namespace
116 
117 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
118   auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
119   if (!run(M, TLI))
120     return PreservedAnalyses::all();
121 
122   return PreservedAnalyses::none();
123 }
124 
125 char InstrProfilingLegacyPass::ID = 0;
126 INITIALIZE_PASS_BEGIN(
127     InstrProfilingLegacyPass, "instrprof",
128     "Frontend instrumentation-based coverage lowering.", false, false)
129 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
130 INITIALIZE_PASS_END(
131     InstrProfilingLegacyPass, "instrprof",
132     "Frontend instrumentation-based coverage lowering.", false, false)
133 
134 ModulePass *
135 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
136   return new InstrProfilingLegacyPass(Options);
137 }
138 
139 bool InstrProfiling::isMachO() const {
140   return Triple(M->getTargetTriple()).isOSBinFormatMachO();
141 }
142 
143 /// Get the section name for the counter variables.
144 StringRef InstrProfiling::getCountersSection() const {
145   return getInstrProfCountersSectionName(isMachO());
146 }
147 
148 /// Get the section name for the name variables.
149 StringRef InstrProfiling::getNameSection() const {
150   return getInstrProfNameSectionName(isMachO());
151 }
152 
153 /// Get the section name for the profile data variables.
154 StringRef InstrProfiling::getDataSection() const {
155   return getInstrProfDataSectionName(isMachO());
156 }
157 
158 /// Get the section name for the coverage mapping data.
159 StringRef InstrProfiling::getCoverageSection() const {
160   return getInstrProfCoverageSectionName(isMachO());
161 }
162 
163 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
164   InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
165   if (Inc)
166     return Inc;
167   return dyn_cast<InstrProfIncrementInst>(Instr);
168 }
169 
170 bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
171   bool MadeChange = false;
172 
173   this->M = &M;
174   this->TLI = &TLI;
175   NamesVar = nullptr;
176   NamesSize = 0;
177   ProfileDataMap.clear();
178   UsedVars.clear();
179   getMemOPSizeOptions();
180 
181   // We did not know how many value sites there would be inside
182   // the instrumented function. This is counting the number of instrumented
183   // target value sites to enter it as field in the profile data variable.
184   for (Function &F : M) {
185     InstrProfIncrementInst *FirstProfIncInst = nullptr;
186     for (BasicBlock &BB : F)
187       for (auto I = BB.begin(), E = BB.end(); I != E; I++)
188         if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
189           computeNumValueSiteCounts(Ind);
190         else if (FirstProfIncInst == nullptr)
191           FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
192 
193     // Value profiling intrinsic lowering requires per-function profile data
194     // variable to be created first.
195     if (FirstProfIncInst != nullptr)
196       static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
197   }
198 
199   for (Function &F : M)
200     for (BasicBlock &BB : F)
201       for (auto I = BB.begin(), E = BB.end(); I != E;) {
202         auto Instr = I++;
203         InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
204         if (Inc) {
205           lowerIncrement(Inc);
206           MadeChange = true;
207         } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
208           lowerValueProfileInst(Ind);
209           MadeChange = true;
210         }
211       }
212 
213   if (GlobalVariable *CoverageNamesVar =
214           M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
215     lowerCoverageData(CoverageNamesVar);
216     MadeChange = true;
217   }
218 
219   if (!MadeChange)
220     return false;
221 
222   emitVNodes();
223   emitNameData();
224   emitRegistration();
225   emitRuntimeHook();
226   emitUses();
227   emitInitialization();
228   return true;
229 }
230 
231 static Constant *getOrInsertValueProfilingCall(Module &M,
232                                                const TargetLibraryInfo &TLI,
233                                                bool IsRange = false) {
234   LLVMContext &Ctx = M.getContext();
235   auto *ReturnTy = Type::getVoidTy(M.getContext());
236 
237   Constant *Res;
238   if (!IsRange) {
239     Type *ParamTypes[] = {
240 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
241 #include "llvm/ProfileData/InstrProfData.inc"
242     };
243     auto *ValueProfilingCallTy =
244         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
245     Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
246                                 ValueProfilingCallTy);
247   } else {
248     Type *RangeParamTypes[] = {
249 #define VALUE_RANGE_PROF 1
250 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
251 #include "llvm/ProfileData/InstrProfData.inc"
252 #undef VALUE_RANGE_PROF
253     };
254     auto *ValueRangeProfilingCallTy =
255         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
256     Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
257                                 ValueRangeProfilingCallTy);
258   }
259 
260   if (Function *FunRes = dyn_cast<Function>(Res)) {
261     if (auto AK = TLI.getExtAttrForI32Param(false))
262       FunRes->addAttribute(3, AK);
263   }
264   return Res;
265 }
266 
267 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
268   GlobalVariable *Name = Ind->getName();
269   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
270   uint64_t Index = Ind->getIndex()->getZExtValue();
271   auto It = ProfileDataMap.find(Name);
272   if (It == ProfileDataMap.end()) {
273     PerFunctionProfileData PD;
274     PD.NumValueSites[ValueKind] = Index + 1;
275     ProfileDataMap[Name] = PD;
276   } else if (It->second.NumValueSites[ValueKind] <= Index)
277     It->second.NumValueSites[ValueKind] = Index + 1;
278 }
279 
280 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
281   GlobalVariable *Name = Ind->getName();
282   auto It = ProfileDataMap.find(Name);
283   assert(It != ProfileDataMap.end() && It->second.DataVar &&
284          "value profiling detected in function with no counter incerement");
285 
286   GlobalVariable *DataVar = It->second.DataVar;
287   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
288   uint64_t Index = Ind->getIndex()->getZExtValue();
289   for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
290     Index += It->second.NumValueSites[Kind];
291 
292   IRBuilder<> Builder(Ind);
293   bool IsRange = (Ind->getValueKind()->getZExtValue() ==
294                   llvm::InstrProfValueKind::IPVK_MemOPSize);
295   CallInst *Call = nullptr;
296   if (!IsRange) {
297     Value *Args[3] = {Ind->getTargetValue(),
298                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
299                       Builder.getInt32(Index)};
300     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
301   } else {
302     Value *Args[6] = {Ind->getTargetValue(),
303                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
304                       Builder.getInt32(Index),
305                       Builder.getInt64(MemOPSizeRangeStart),
306                       Builder.getInt64(MemOPSizeRangeLast),
307                       Builder.getInt64(MemOPSizeLargeVal)};
308     Call =
309         Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
310   }
311   if (auto AK = TLI->getExtAttrForI32Param(false))
312     Call->addAttribute(3, AK);
313   Ind->replaceAllUsesWith(Call);
314   Ind->eraseFromParent();
315 }
316 
317 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
318   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
319 
320   IRBuilder<> Builder(Inc);
321   uint64_t Index = Inc->getIndex()->getZExtValue();
322   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
323   Value *Count = Builder.CreateLoad(Addr, "pgocount");
324   Count = Builder.CreateAdd(Count, Inc->getStep());
325   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
326   Inc->eraseFromParent();
327 }
328 
329 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
330   ConstantArray *Names =
331       cast<ConstantArray>(CoverageNamesVar->getInitializer());
332   for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
333     Constant *NC = Names->getOperand(I);
334     Value *V = NC->stripPointerCasts();
335     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
336     GlobalVariable *Name = cast<GlobalVariable>(V);
337 
338     Name->setLinkage(GlobalValue::PrivateLinkage);
339     ReferencedNames.push_back(Name);
340     NC->dropAllReferences();
341   }
342   CoverageNamesVar->eraseFromParent();
343 }
344 
345 /// Get the name of a profiling variable for a particular function.
346 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
347   StringRef NamePrefix = getInstrProfNameVarPrefix();
348   StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
349   Function *F = Inc->getParent()->getParent();
350   Module *M = F->getParent();
351   if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
352       !canRenameComdatFunc(*F))
353     return (Prefix + Name).str();
354   uint64_t FuncHash = Inc->getHash()->getZExtValue();
355   SmallVector<char, 24> HashPostfix;
356   if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
357     return (Prefix + Name).str();
358   return (Prefix + Name + "." + Twine(FuncHash)).str();
359 }
360 
361 static inline bool shouldRecordFunctionAddr(Function *F) {
362   // Check the linkage
363   if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
364       !F->hasAvailableExternallyLinkage())
365     return true;
366   // Prohibit function address recording if the function is both internal and
367   // COMDAT. This avoids the profile data variable referencing internal symbols
368   // in COMDAT.
369   if (F->hasLocalLinkage() && F->hasComdat())
370     return false;
371   // Check uses of this function for other than direct calls or invokes to it.
372   // Inline virtual functions have linkeOnceODR linkage. When a key method
373   // exists, the vtable will only be emitted in the TU where the key method
374   // is defined. In a TU where vtable is not available, the function won't
375   // be 'addresstaken'. If its address is not recorded here, the profile data
376   // with missing address may be picked by the linker leading  to missing
377   // indirect call target info.
378   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
379 }
380 
381 static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
382                                                InstrProfIncrementInst *Inc) {
383   if (!needsComdatForCounter(F, M))
384     return nullptr;
385 
386   // COFF format requires a COMDAT section to have a key symbol with the same
387   // name. The linker targeting COFF also requires that the COMDAT
388   // a section is associated to must precede the associating section. For this
389   // reason, we must choose the counter var's name as the name of the comdat.
390   StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
391                                 ? getInstrProfCountersVarPrefix()
392                                 : getInstrProfComdatPrefix());
393   return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
394 }
395 
396 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
397   // Don't do this for Darwin.  compiler-rt uses linker magic.
398   if (Triple(M.getTargetTriple()).isOSDarwin())
399     return false;
400 
401   // Use linker script magic to get data/cnts/name start/end.
402   if (Triple(M.getTargetTriple()).isOSLinux() ||
403       Triple(M.getTargetTriple()).isOSFreeBSD() ||
404       Triple(M.getTargetTriple()).isPS4CPU())
405     return false;
406 
407   return true;
408 }
409 
410 GlobalVariable *
411 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
412   GlobalVariable *NamePtr = Inc->getName();
413   auto It = ProfileDataMap.find(NamePtr);
414   PerFunctionProfileData PD;
415   if (It != ProfileDataMap.end()) {
416     if (It->second.RegionCounters)
417       return It->second.RegionCounters;
418     PD = It->second;
419   }
420 
421   // Move the name variable to the right section. Place them in a COMDAT group
422   // if the associated function is a COMDAT. This will make sure that
423   // only one copy of counters of the COMDAT function will be emitted after
424   // linking.
425   Function *Fn = Inc->getParent()->getParent();
426   Comdat *ProfileVarsComdat = nullptr;
427   ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
428 
429   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
430   LLVMContext &Ctx = M->getContext();
431   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
432 
433   // Create the counters variable.
434   auto *CounterPtr =
435       new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
436                          Constant::getNullValue(CounterTy),
437                          getVarName(Inc, getInstrProfCountersVarPrefix()));
438   CounterPtr->setVisibility(NamePtr->getVisibility());
439   CounterPtr->setSection(getCountersSection());
440   CounterPtr->setAlignment(8);
441   CounterPtr->setComdat(ProfileVarsComdat);
442 
443   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
444   // Allocate statically the array of pointers to value profile nodes for
445   // the current function.
446   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
447   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
448     uint64_t NS = 0;
449     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
450       NS += PD.NumValueSites[Kind];
451     if (NS) {
452       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
453 
454       auto *ValuesVar =
455           new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
456                              Constant::getNullValue(ValuesTy),
457                              getVarName(Inc, getInstrProfValuesVarPrefix()));
458       ValuesVar->setVisibility(NamePtr->getVisibility());
459       ValuesVar->setSection(getInstrProfValuesSectionName(isMachO()));
460       ValuesVar->setAlignment(8);
461       ValuesVar->setComdat(ProfileVarsComdat);
462       ValuesPtrExpr =
463           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
464     }
465   }
466 
467   // Create data variable.
468   auto *Int16Ty = Type::getInt16Ty(Ctx);
469   auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
470   Type *DataTypes[] = {
471 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
472 #include "llvm/ProfileData/InstrProfData.inc"
473   };
474   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
475 
476   Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
477                                ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
478                                : ConstantPointerNull::get(Int8PtrTy);
479 
480   Constant *Int16ArrayVals[IPVK_Last + 1];
481   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
482     Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
483 
484   Constant *DataVals[] = {
485 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
486 #include "llvm/ProfileData/InstrProfData.inc"
487   };
488   auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
489                                   ConstantStruct::get(DataTy, DataVals),
490                                   getVarName(Inc, getInstrProfDataVarPrefix()));
491   Data->setVisibility(NamePtr->getVisibility());
492   Data->setSection(getDataSection());
493   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
494   Data->setComdat(ProfileVarsComdat);
495 
496   PD.RegionCounters = CounterPtr;
497   PD.DataVar = Data;
498   ProfileDataMap[NamePtr] = PD;
499 
500   // Mark the data variable as used so that it isn't stripped out.
501   UsedVars.push_back(Data);
502   // Now that the linkage set by the FE has been passed to the data and counter
503   // variables, reset Name variable's linkage and visibility to private so that
504   // it can be removed later by the compiler.
505   NamePtr->setLinkage(GlobalValue::PrivateLinkage);
506   // Collect the referenced names to be used by emitNameData.
507   ReferencedNames.push_back(NamePtr);
508 
509   return CounterPtr;
510 }
511 
512 void InstrProfiling::emitVNodes() {
513   if (!ValueProfileStaticAlloc)
514     return;
515 
516   // For now only support this on platforms that do
517   // not require runtime registration to discover
518   // named section start/end.
519   if (needsRuntimeRegistrationOfSectionRange(*M))
520     return;
521 
522   size_t TotalNS = 0;
523   for (auto &PD : ProfileDataMap) {
524     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
525       TotalNS += PD.second.NumValueSites[Kind];
526   }
527 
528   if (!TotalNS)
529     return;
530 
531   uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
532 // Heuristic for small programs with very few total value sites.
533 // The default value of vp-counters-per-site is chosen based on
534 // the observation that large apps usually have a low percentage
535 // of value sites that actually have any profile data, and thus
536 // the average number of counters per site is low. For small
537 // apps with very few sites, this may not be true. Bump up the
538 // number of counters in this case.
539 #define INSTR_PROF_MIN_VAL_COUNTS 10
540   if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
541     NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
542 
543   auto &Ctx = M->getContext();
544   Type *VNodeTypes[] = {
545 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
546 #include "llvm/ProfileData/InstrProfData.inc"
547   };
548   auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
549 
550   ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
551   auto *VNodesVar = new GlobalVariable(
552       *M, VNodesTy, false, GlobalValue::PrivateLinkage,
553       Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
554   VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO()));
555   UsedVars.push_back(VNodesVar);
556 }
557 
558 void InstrProfiling::emitNameData() {
559   std::string UncompressedData;
560 
561   if (ReferencedNames.empty())
562     return;
563 
564   std::string CompressedNameStr;
565   if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
566                                           DoNameCompression)) {
567     report_fatal_error(toString(std::move(E)), false);
568   }
569 
570   auto &Ctx = M->getContext();
571   auto *NamesVal = ConstantDataArray::getString(
572       Ctx, StringRef(CompressedNameStr), false);
573   NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
574                                 GlobalValue::PrivateLinkage, NamesVal,
575                                 getInstrProfNamesVarName());
576   NamesSize = CompressedNameStr.size();
577   NamesVar->setSection(getNameSection());
578   UsedVars.push_back(NamesVar);
579 
580   for (auto *NamePtr : ReferencedNames)
581     NamePtr->eraseFromParent();
582 }
583 
584 void InstrProfiling::emitRegistration() {
585   if (!needsRuntimeRegistrationOfSectionRange(*M))
586     return;
587 
588   // Construct the function.
589   auto *VoidTy = Type::getVoidTy(M->getContext());
590   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
591   auto *Int64Ty = Type::getInt64Ty(M->getContext());
592   auto *RegisterFTy = FunctionType::get(VoidTy, false);
593   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
594                                      getInstrProfRegFuncsName(), M);
595   RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
596   if (Options.NoRedZone)
597     RegisterF->addFnAttr(Attribute::NoRedZone);
598 
599   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
600   auto *RuntimeRegisterF =
601       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
602                        getInstrProfRegFuncName(), M);
603 
604   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
605   for (Value *Data : UsedVars)
606     if (Data != NamesVar)
607       IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
608 
609   if (NamesVar) {
610     Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
611     auto *NamesRegisterTy =
612         FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
613     auto *NamesRegisterF =
614         Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
615                          getInstrProfNamesRegFuncName(), M);
616     IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
617                                     IRB.getInt64(NamesSize)});
618   }
619 
620   IRB.CreateRetVoid();
621 }
622 
623 void InstrProfiling::emitRuntimeHook() {
624   // We expect the linker to be invoked with -u<hook_var> flag for linux,
625   // for which case there is no need to emit the user function.
626   if (Triple(M->getTargetTriple()).isOSLinux())
627     return;
628 
629   // If the module's provided its own runtime, we don't need to do anything.
630   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
631     return;
632 
633   // Declare an external variable that will pull in the runtime initialization.
634   auto *Int32Ty = Type::getInt32Ty(M->getContext());
635   auto *Var =
636       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
637                          nullptr, getInstrProfRuntimeHookVarName());
638 
639   // Make a function that uses it.
640   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
641                                 GlobalValue::LinkOnceODRLinkage,
642                                 getInstrProfRuntimeHookVarUseFuncName(), M);
643   User->addFnAttr(Attribute::NoInline);
644   if (Options.NoRedZone)
645     User->addFnAttr(Attribute::NoRedZone);
646   User->setVisibility(GlobalValue::HiddenVisibility);
647   if (Triple(M->getTargetTriple()).supportsCOMDAT())
648     User->setComdat(M->getOrInsertComdat(User->getName()));
649 
650   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
651   auto *Load = IRB.CreateLoad(Var);
652   IRB.CreateRet(Load);
653 
654   // Mark the user variable as used so that it isn't stripped out.
655   UsedVars.push_back(User);
656 }
657 
658 void InstrProfiling::emitUses() {
659   if (!UsedVars.empty())
660     appendToUsed(*M, UsedVars);
661 }
662 
663 void InstrProfiling::emitInitialization() {
664   StringRef InstrProfileOutput = Options.InstrProfileOutput;
665 
666   if (!InstrProfileOutput.empty()) {
667     // Create variable for profile name.
668     Constant *ProfileNameConst =
669         ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
670     GlobalVariable *ProfileNameVar = new GlobalVariable(
671         *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
672         ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
673     Triple TT(M->getTargetTriple());
674     if (TT.supportsCOMDAT()) {
675       ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
676       ProfileNameVar->setComdat(M->getOrInsertComdat(
677           StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
678     }
679   }
680 
681   Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
682   if (!RegisterF)
683     return;
684 
685   // Create the initialization function.
686   auto *VoidTy = Type::getVoidTy(M->getContext());
687   auto *F = Function::Create(FunctionType::get(VoidTy, false),
688                              GlobalValue::InternalLinkage,
689                              getInstrProfInitFuncName(), M);
690   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
691   F->addFnAttr(Attribute::NoInline);
692   if (Options.NoRedZone)
693     F->addFnAttr(Attribute::NoRedZone);
694 
695   // Add the basic block and the necessary calls.
696   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
697   if (RegisterF)
698     IRB.CreateCall(RegisterF, {});
699   IRB.CreateRetVoid();
700 
701   appendToGlobalCtors(*M, F, 0);
702 }
703 
704 void InstrProfiling::getMemOPSizeOptions() {
705   // Parse the value profile options.
706   MemOPSizeRangeStart = DefaultMemOPSizeRangeStart;
707   MemOPSizeRangeLast = DefaultMemOPSizeRangeLast;
708   if (!MemOPSizeRange.empty()) {
709     auto Pos = MemOPSizeRange.find(":");
710     if (Pos != std::string::npos) {
711       if (Pos > 0)
712         MemOPSizeRangeStart = atoi(MemOPSizeRange.substr(0, Pos).c_str());
713       if (Pos < MemOPSizeRange.size() - 1)
714         MemOPSizeRangeLast = atoi(MemOPSizeRange.substr(Pos + 1).c_str());
715     } else
716       MemOPSizeRangeLast = atoi(MemOPSizeRange.c_str());
717   }
718   assert(MemOPSizeRangeLast >= MemOPSizeRangeStart);
719 
720   MemOPSizeLargeVal = MemOPSizeLarge;
721   if (MemOPSizeLargeVal == 0)
722     MemOPSizeLargeVal = INT64_MIN;
723 }
724