1 //=-- InstrProf.cpp - Instrumented profiling format support -----------------=//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains support for clang's instrumentation based PGO and
11 // coverage.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/ProfileData/InstrProf.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/Function.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/MDBuilder.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Support/Compression.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/LEB128.h"
25 #include "llvm/Support/ManagedStatic.h"
26 
27 using namespace llvm;
28 
29 namespace {
30 class InstrProfErrorCategoryType : public std::error_category {
31   const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; }
32   std::string message(int IE) const override {
33     instrprof_error E = static_cast<instrprof_error>(IE);
34     switch (E) {
35     case instrprof_error::success:
36       return "Success";
37     case instrprof_error::eof:
38       return "End of File";
39     case instrprof_error::unrecognized_format:
40       return "Unrecognized instrumentation profile encoding format";
41     case instrprof_error::bad_magic:
42       return "Invalid instrumentation profile data (bad magic)";
43     case instrprof_error::bad_header:
44       return "Invalid instrumentation profile data (file header is corrupt)";
45     case instrprof_error::unsupported_version:
46       return "Unsupported instrumentation profile format version";
47     case instrprof_error::unsupported_hash_type:
48       return "Unsupported instrumentation profile hash type";
49     case instrprof_error::too_large:
50       return "Too much profile data";
51     case instrprof_error::truncated:
52       return "Truncated profile data";
53     case instrprof_error::malformed:
54       return "Malformed instrumentation profile data";
55     case instrprof_error::unknown_function:
56       return "No profile data available for function";
57     case instrprof_error::hash_mismatch:
58       return "Function control flow change detected (hash mismatch)";
59     case instrprof_error::count_mismatch:
60       return "Function basic block count change detected (counter mismatch)";
61     case instrprof_error::counter_overflow:
62       return "Counter overflow";
63     case instrprof_error::value_site_count_mismatch:
64       return "Function value site count change detected (counter mismatch)";
65     case instrprof_error::compress_failed:
66       return "Failed to compress data (zlib)";
67     case instrprof_error::uncompress_failed:
68       return "Failed to uncompress data (zlib)";
69     }
70     llvm_unreachable("A value of instrprof_error has no message.");
71   }
72 };
73 } // end anonymous namespace
74 
75 static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
76 
77 const std::error_category &llvm::instrprof_category() {
78   return *ErrorCategory;
79 }
80 
81 namespace llvm {
82 
83 std::string getPGOFuncName(StringRef RawFuncName,
84                            GlobalValue::LinkageTypes Linkage,
85                            StringRef FileName,
86                            uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
87   return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
88 }
89 
90 // Return the PGOFuncName. This function has some special handling when called
91 // in LTO optimization. The following only applies when calling in LTO passes
92 // (when \c InLTO is true): LTO's internalization privatizes many global linkage
93 // symbols. This happens after value profile annotation, but those internal
94 // linkage functions should not have a source prefix.
95 // To differentiate compiler generated internal symbols from original ones,
96 // PGOFuncName meta data are created and attached to the original internal
97 // symbols in the value profile annotation step
98 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
99 // data, its original linkage must be non-internal.
100 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
101   if (!InLTO)
102     return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(),
103                           Version);
104 
105   // In LTO mode (when InLTO is true), first check if there is a meta data.
106   if (MDNode *MD = getPGOFuncNameMetadata(F)) {
107     StringRef S = cast<MDString>(MD->getOperand(0))->getString();
108     return S.str();
109   }
110 
111   // If there is no meta data, the function must be a global before the value
112   // profile annotation pass. Its current linkage may be internal if it is
113   // internalized in LTO mode.
114   return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
115 }
116 
117 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
118   if (FileName.empty())
119     return PGOFuncName;
120   // Drop the file name including ':'. See also getPGOFuncName.
121   if (PGOFuncName.startswith(FileName))
122     PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
123   return PGOFuncName;
124 }
125 
126 // \p FuncName is the string used as profile lookup key for the function. A
127 // symbol is created to hold the name. Return the legalized symbol name.
128 std::string getPGOFuncNameVarName(StringRef FuncName,
129                                   GlobalValue::LinkageTypes Linkage) {
130   std::string VarName = getInstrProfNameVarPrefix();
131   VarName += FuncName;
132 
133   if (!GlobalValue::isLocalLinkage(Linkage))
134     return VarName;
135 
136   // Now fix up illegal chars in local VarName that may upset the assembler.
137   const char *InvalidChars = "-:<>\"'";
138   size_t found = VarName.find_first_of(InvalidChars);
139   while (found != std::string::npos) {
140     VarName[found] = '_';
141     found = VarName.find_first_of(InvalidChars, found + 1);
142   }
143   return VarName;
144 }
145 
146 GlobalVariable *createPGOFuncNameVar(Module &M,
147                                      GlobalValue::LinkageTypes Linkage,
148                                      StringRef PGOFuncName) {
149 
150   // We generally want to match the function's linkage, but available_externally
151   // and extern_weak both have the wrong semantics, and anything that doesn't
152   // need to link across compilation units doesn't need to be visible at all.
153   if (Linkage == GlobalValue::ExternalWeakLinkage)
154     Linkage = GlobalValue::LinkOnceAnyLinkage;
155   else if (Linkage == GlobalValue::AvailableExternallyLinkage)
156     Linkage = GlobalValue::LinkOnceODRLinkage;
157   else if (Linkage == GlobalValue::InternalLinkage ||
158            Linkage == GlobalValue::ExternalLinkage)
159     Linkage = GlobalValue::PrivateLinkage;
160 
161   auto *Value =
162       ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
163   auto FuncNameVar =
164       new GlobalVariable(M, Value->getType(), true, Linkage, Value,
165                          getPGOFuncNameVarName(PGOFuncName, Linkage));
166 
167   // Hide the symbol so that we correctly get a copy for each executable.
168   if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
169     FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
170 
171   return FuncNameVar;
172 }
173 
174 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
175   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
176 }
177 
178 void InstrProfSymtab::create(Module &M, bool InLTO) {
179   for (Function &F : M) {
180     // Function may not have a name: like using asm("") to overwrite the name.
181     // Ignore in this case.
182     if (!F.hasName())
183       continue;
184     const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
185     addFuncName(PGOFuncName);
186     MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
187   }
188 
189   finalizeSymtab();
190 }
191 
192 std::error_code
193 collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs,
194                           bool doCompression, std::string &Result) {
195   assert(NameStrs.size() && "No name data to emit");
196 
197   uint8_t Header[16], *P = Header;
198   std::string UncompressedNameStrings =
199       join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
200 
201   assert(StringRef(UncompressedNameStrings)
202                  .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
203          "PGO name is invalid (contains separator token)");
204 
205   unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
206   P += EncLen;
207 
208   auto WriteStringToResult = [&](size_t CompressedLen,
209                                  const std::string &InputStr) {
210     EncLen = encodeULEB128(CompressedLen, P);
211     P += EncLen;
212     char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
213     unsigned HeaderLen = P - &Header[0];
214     Result.append(HeaderStr, HeaderLen);
215     Result += InputStr;
216     return make_error_code(instrprof_error::success);
217   };
218 
219   if (!doCompression)
220     return WriteStringToResult(0, UncompressedNameStrings);
221 
222   SmallVector<char, 128> CompressedNameStrings;
223   zlib::Status Success =
224       zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings,
225                      zlib::BestSizeCompression);
226 
227   if (Success != zlib::StatusOK)
228     return make_error_code(instrprof_error::compress_failed);
229 
230   return WriteStringToResult(
231       CompressedNameStrings.size(),
232       std::string(CompressedNameStrings.data(), CompressedNameStrings.size()));
233 }
234 
235 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
236   auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
237   StringRef NameStr =
238       Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
239   return NameStr;
240 }
241 
242 std::error_code
243 collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars,
244                           std::string &Result, bool doCompression) {
245   std::vector<std::string> NameStrs;
246   for (auto *NameVar : NameVars) {
247     NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar));
248   }
249   return collectPGOFuncNameStrings(
250       NameStrs, zlib::isAvailable() && doCompression, Result);
251 }
252 
253 std::error_code readPGOFuncNameStrings(StringRef NameStrings,
254                                        InstrProfSymtab &Symtab) {
255   const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data());
256   const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() +
257                                                           NameStrings.size());
258   while (P < EndP) {
259     uint32_t N;
260     uint64_t UncompressedSize = decodeULEB128(P, &N);
261     P += N;
262     uint64_t CompressedSize = decodeULEB128(P, &N);
263     P += N;
264     bool isCompressed = (CompressedSize != 0);
265     SmallString<128> UncompressedNameStrings;
266     StringRef NameStrings;
267     if (isCompressed) {
268       StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
269                                       CompressedSize);
270       if (zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
271                            UncompressedSize) != zlib::StatusOK)
272         return make_error_code(instrprof_error::uncompress_failed);
273       P += CompressedSize;
274       NameStrings = StringRef(UncompressedNameStrings.data(),
275                               UncompressedNameStrings.size());
276     } else {
277       NameStrings =
278           StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
279       P += UncompressedSize;
280     }
281     // Now parse the name strings.
282     SmallVector<StringRef, 0> Names;
283     NameStrings.split(Names, getInstrProfNameSeparator());
284     for (StringRef &Name : Names)
285       Symtab.addFuncName(Name);
286 
287     while (P < EndP && *P == 0)
288       P++;
289   }
290   Symtab.finalizeSymtab();
291   return make_error_code(instrprof_error::success);
292 }
293 
294 instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
295                                                 uint64_t Weight) {
296   this->sortByTargetValues();
297   Input.sortByTargetValues();
298   auto I = ValueData.begin();
299   auto IE = ValueData.end();
300   instrprof_error Result = instrprof_error::success;
301   for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
302        ++J) {
303     while (I != IE && I->Value < J->Value)
304       ++I;
305     if (I != IE && I->Value == J->Value) {
306       bool Overflowed;
307       I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
308       if (Overflowed)
309         Result = instrprof_error::counter_overflow;
310       ++I;
311       continue;
312     }
313     ValueData.insert(I, *J);
314   }
315   return Result;
316 }
317 
318 instrprof_error InstrProfValueSiteRecord::scale(uint64_t Weight) {
319   instrprof_error Result = instrprof_error::success;
320   for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
321     bool Overflowed;
322     I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
323     if (Overflowed)
324       Result = instrprof_error::counter_overflow;
325   }
326   return Result;
327 }
328 
329 // Merge Value Profile data from Src record to this record for ValueKind.
330 // Scale merged value counts by \p Weight.
331 instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
332                                                     InstrProfRecord &Src,
333                                                     uint64_t Weight) {
334   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
335   uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
336   if (ThisNumValueSites != OtherNumValueSites)
337     return instrprof_error::value_site_count_mismatch;
338   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
339       getValueSitesForKind(ValueKind);
340   std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
341       Src.getValueSitesForKind(ValueKind);
342   instrprof_error Result = instrprof_error::success;
343   for (uint32_t I = 0; I < ThisNumValueSites; I++)
344     MergeResult(Result, ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight));
345   return Result;
346 }
347 
348 instrprof_error InstrProfRecord::merge(InstrProfRecord &Other,
349                                        uint64_t Weight) {
350   // If the number of counters doesn't match we either have bad data
351   // or a hash collision.
352   if (Counts.size() != Other.Counts.size())
353     return instrprof_error::count_mismatch;
354 
355   instrprof_error Result = instrprof_error::success;
356 
357   for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
358     bool Overflowed;
359     Counts[I] =
360         SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
361     if (Overflowed)
362       Result = instrprof_error::counter_overflow;
363   }
364 
365   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
366     MergeResult(Result, mergeValueProfData(Kind, Other, Weight));
367 
368   return Result;
369 }
370 
371 instrprof_error InstrProfRecord::scaleValueProfData(uint32_t ValueKind,
372                                                     uint64_t Weight) {
373   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
374   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
375       getValueSitesForKind(ValueKind);
376   instrprof_error Result = instrprof_error::success;
377   for (uint32_t I = 0; I < ThisNumValueSites; I++)
378     MergeResult(Result, ThisSiteRecords[I].scale(Weight));
379   return Result;
380 }
381 
382 instrprof_error InstrProfRecord::scale(uint64_t Weight) {
383   instrprof_error Result = instrprof_error::success;
384   for (auto &Count : this->Counts) {
385     bool Overflowed;
386     Count = SaturatingMultiply(Count, Weight, &Overflowed);
387     if (Overflowed && Result == instrprof_error::success) {
388       Result = instrprof_error::counter_overflow;
389     }
390   }
391   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
392     MergeResult(Result, scaleValueProfData(Kind, Weight));
393 
394   return Result;
395 }
396 
397 // Map indirect call target name hash to name string.
398 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
399                                      ValueMapType *ValueMap) {
400   if (!ValueMap)
401     return Value;
402   switch (ValueKind) {
403   case IPVK_IndirectCallTarget: {
404     auto Result =
405         std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
406                          [](const std::pair<uint64_t, uint64_t> &LHS,
407                             uint64_t RHS) { return LHS.first < RHS; });
408    // Raw function pointer collected by value profiler may be from
409    // external functions that are not instrumented. They won't have
410    // mapping data to be used by the deserializer. Force the value to
411    // be 0 in this case.
412     if (Result != ValueMap->end() && Result->first == Value)
413       Value = (uint64_t)Result->second;
414     else
415       Value = 0;
416     break;
417   }
418   }
419   return Value;
420 }
421 
422 void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
423                                    InstrProfValueData *VData, uint32_t N,
424                                    ValueMapType *ValueMap) {
425   for (uint32_t I = 0; I < N; I++) {
426     VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
427   }
428   std::vector<InstrProfValueSiteRecord> &ValueSites =
429       getValueSitesForKind(ValueKind);
430   if (N == 0)
431     ValueSites.push_back(InstrProfValueSiteRecord());
432   else
433     ValueSites.emplace_back(VData, VData + N);
434 }
435 
436 #define INSTR_PROF_COMMON_API_IMPL
437 #include "llvm/ProfileData/InstrProfData.inc"
438 
439 /*!
440  * \brief ValueProfRecordClosure Interface implementation for  InstrProfRecord
441  *  class. These C wrappers are used as adaptors so that C++ code can be
442  *  invoked as callbacks.
443  */
444 uint32_t getNumValueKindsInstrProf(const void *Record) {
445   return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
446 }
447 
448 uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
449   return reinterpret_cast<const InstrProfRecord *>(Record)
450       ->getNumValueSites(VKind);
451 }
452 
453 uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
454   return reinterpret_cast<const InstrProfRecord *>(Record)
455       ->getNumValueData(VKind);
456 }
457 
458 uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
459                                          uint32_t S) {
460   return reinterpret_cast<const InstrProfRecord *>(R)
461       ->getNumValueDataForSite(VK, S);
462 }
463 
464 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
465                               uint32_t K, uint32_t S) {
466   reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
467 }
468 
469 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
470   ValueProfData *VD =
471       (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
472   memset(VD, 0, TotalSizeInBytes);
473   return VD;
474 }
475 
476 static ValueProfRecordClosure InstrProfRecordClosure = {
477     nullptr,
478     getNumValueKindsInstrProf,
479     getNumValueSitesInstrProf,
480     getNumValueDataInstrProf,
481     getNumValueDataForSiteInstrProf,
482     nullptr,
483     getValueForSiteInstrProf,
484     allocValueProfDataInstrProf};
485 
486 // Wrapper implementation using the closure mechanism.
487 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
488   InstrProfRecordClosure.Record = &Record;
489   return getValueProfDataSize(&InstrProfRecordClosure);
490 }
491 
492 // Wrapper implementation using the closure mechanism.
493 std::unique_ptr<ValueProfData>
494 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
495   InstrProfRecordClosure.Record = &Record;
496 
497   std::unique_ptr<ValueProfData> VPD(
498       serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
499   return VPD;
500 }
501 
502 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
503                                     InstrProfRecord::ValueMapType *VMap) {
504   Record.reserveSites(Kind, NumValueSites);
505 
506   InstrProfValueData *ValueData = getValueProfRecordValueData(this);
507   for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
508     uint8_t ValueDataCount = this->SiteCountArray[VSite];
509     Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap);
510     ValueData += ValueDataCount;
511   }
512 }
513 
514 // For writing/serializing,  Old is the host endianness, and  New is
515 // byte order intended on disk. For Reading/deserialization, Old
516 // is the on-disk source endianness, and New is the host endianness.
517 void ValueProfRecord::swapBytes(support::endianness Old,
518                                 support::endianness New) {
519   using namespace support;
520   if (Old == New)
521     return;
522 
523   if (getHostEndianness() != Old) {
524     sys::swapByteOrder<uint32_t>(NumValueSites);
525     sys::swapByteOrder<uint32_t>(Kind);
526   }
527   uint32_t ND = getValueProfRecordNumValueData(this);
528   InstrProfValueData *VD = getValueProfRecordValueData(this);
529 
530   // No need to swap byte array: SiteCountArrray.
531   for (uint32_t I = 0; I < ND; I++) {
532     sys::swapByteOrder<uint64_t>(VD[I].Value);
533     sys::swapByteOrder<uint64_t>(VD[I].Count);
534   }
535   if (getHostEndianness() == Old) {
536     sys::swapByteOrder<uint32_t>(NumValueSites);
537     sys::swapByteOrder<uint32_t>(Kind);
538   }
539 }
540 
541 void ValueProfData::deserializeTo(InstrProfRecord &Record,
542                                   InstrProfRecord::ValueMapType *VMap) {
543   if (NumValueKinds == 0)
544     return;
545 
546   ValueProfRecord *VR = getFirstValueProfRecord(this);
547   for (uint32_t K = 0; K < NumValueKinds; K++) {
548     VR->deserializeTo(Record, VMap);
549     VR = getValueProfRecordNext(VR);
550   }
551 }
552 
553 template <class T>
554 static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
555   using namespace support;
556   if (Orig == little)
557     return endian::readNext<T, little, unaligned>(D);
558   else
559     return endian::readNext<T, big, unaligned>(D);
560 }
561 
562 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
563   return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
564                                             ValueProfData());
565 }
566 
567 instrprof_error ValueProfData::checkIntegrity() {
568   if (NumValueKinds > IPVK_Last + 1)
569     return instrprof_error::malformed;
570   // Total size needs to be mulltiple of quadword size.
571   if (TotalSize % sizeof(uint64_t))
572     return instrprof_error::malformed;
573 
574   ValueProfRecord *VR = getFirstValueProfRecord(this);
575   for (uint32_t K = 0; K < this->NumValueKinds; K++) {
576     if (VR->Kind > IPVK_Last)
577       return instrprof_error::malformed;
578     VR = getValueProfRecordNext(VR);
579     if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
580       return instrprof_error::malformed;
581   }
582   return instrprof_error::success;
583 }
584 
585 ErrorOr<std::unique_ptr<ValueProfData>>
586 ValueProfData::getValueProfData(const unsigned char *D,
587                                 const unsigned char *const BufferEnd,
588                                 support::endianness Endianness) {
589   using namespace support;
590   if (D + sizeof(ValueProfData) > BufferEnd)
591     return instrprof_error::truncated;
592 
593   const unsigned char *Header = D;
594   uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
595   if (D + TotalSize > BufferEnd)
596     return instrprof_error::too_large;
597 
598   std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
599   memcpy(VPD.get(), D, TotalSize);
600   // Byte swap.
601   VPD->swapBytesToHost(Endianness);
602 
603   instrprof_error EC = VPD->checkIntegrity();
604   if (EC != instrprof_error::success)
605     return EC;
606 
607   return std::move(VPD);
608 }
609 
610 void ValueProfData::swapBytesToHost(support::endianness Endianness) {
611   using namespace support;
612   if (Endianness == getHostEndianness())
613     return;
614 
615   sys::swapByteOrder<uint32_t>(TotalSize);
616   sys::swapByteOrder<uint32_t>(NumValueKinds);
617 
618   ValueProfRecord *VR = getFirstValueProfRecord(this);
619   for (uint32_t K = 0; K < NumValueKinds; K++) {
620     VR->swapBytes(Endianness, getHostEndianness());
621     VR = getValueProfRecordNext(VR);
622   }
623 }
624 
625 void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
626   using namespace support;
627   if (Endianness == getHostEndianness())
628     return;
629 
630   ValueProfRecord *VR = getFirstValueProfRecord(this);
631   for (uint32_t K = 0; K < NumValueKinds; K++) {
632     ValueProfRecord *NVR = getValueProfRecordNext(VR);
633     VR->swapBytes(getHostEndianness(), Endianness);
634     VR = NVR;
635   }
636   sys::swapByteOrder<uint32_t>(TotalSize);
637   sys::swapByteOrder<uint32_t>(NumValueKinds);
638 }
639 
640 void annotateValueSite(Module &M, Instruction &Inst,
641                        const InstrProfRecord &InstrProfR,
642                        InstrProfValueKind ValueKind, uint32_t SiteIdx,
643                        uint32_t MaxMDCount) {
644   uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
645   if (!NV)
646     return;
647 
648   uint64_t Sum = 0;
649   std::unique_ptr<InstrProfValueData[]> VD =
650       InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
651 
652   ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
653   annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
654 }
655 
656 void annotateValueSite(Module &M, Instruction &Inst,
657                        ArrayRef<InstrProfValueData> VDs,
658                        uint64_t Sum, InstrProfValueKind ValueKind,
659                        uint32_t MaxMDCount) {
660   LLVMContext &Ctx = M.getContext();
661   MDBuilder MDHelper(Ctx);
662   SmallVector<Metadata *, 3> Vals;
663   // Tag
664   Vals.push_back(MDHelper.createString("VP"));
665   // Value Kind
666   Vals.push_back(MDHelper.createConstant(
667       ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
668   // Total Count
669   Vals.push_back(
670       MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
671 
672   // Value Profile Data
673   uint32_t MDCount = MaxMDCount;
674   for (auto &VD : VDs) {
675     Vals.push_back(MDHelper.createConstant(
676         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
677     Vals.push_back(MDHelper.createConstant(
678         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
679     if (--MDCount == 0)
680       break;
681   }
682   Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
683 }
684 
685 bool getValueProfDataFromInst(const Instruction &Inst,
686                               InstrProfValueKind ValueKind,
687                               uint32_t MaxNumValueData,
688                               InstrProfValueData ValueData[],
689                               uint32_t &ActualNumValueData, uint64_t &TotalC) {
690   MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
691   if (!MD)
692     return false;
693 
694   unsigned NOps = MD->getNumOperands();
695 
696   if (NOps < 5)
697     return false;
698 
699   // Operand 0 is a string tag "VP":
700   MDString *Tag = cast<MDString>(MD->getOperand(0));
701   if (!Tag)
702     return false;
703 
704   if (!Tag->getString().equals("VP"))
705     return false;
706 
707   // Now check kind:
708   ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
709   if (!KindInt)
710     return false;
711   if (KindInt->getZExtValue() != ValueKind)
712     return false;
713 
714   // Get total count
715   ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
716   if (!TotalCInt)
717     return false;
718   TotalC = TotalCInt->getZExtValue();
719 
720   ActualNumValueData = 0;
721 
722   for (unsigned I = 3; I < NOps; I += 2) {
723     if (ActualNumValueData >= MaxNumValueData)
724       break;
725     ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
726     ConstantInt *Count =
727         mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
728     if (!Value || !Count)
729       return false;
730     ValueData[ActualNumValueData].Value = Value->getZExtValue();
731     ValueData[ActualNumValueData].Count = Count->getZExtValue();
732     ActualNumValueData++;
733   }
734   return true;
735 }
736 
737 MDNode *getPGOFuncNameMetadata(const Function &F) {
738   return F.getMetadata(getPGOFuncNameMetadataName());
739 }
740 
741 void createPGOFuncNameMetadata(Function &F, const std::string &PGOFuncName) {
742   // Only for internal linkage functions.
743   if (PGOFuncName == F.getName())
744       return;
745   // Don't create duplicated meta-data.
746   if (getPGOFuncNameMetadata(F))
747     return;
748   LLVMContext &C = F.getContext();
749   MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName.c_str()));
750   F.setMetadata(getPGOFuncNameMetadataName(), N);
751 }
752 
753 } // end namespace llvm
754