1 //=-- SampleProf.cpp - Sample profiling format support --------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains common definitions used in the reading and writing of
10 // sample profile data.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ProfileData/SampleProf.h"
15 #include "llvm/Config/llvm-config.h"
16 #include "llvm/IR/DebugInfoMetadata.h"
17 #include "llvm/IR/PseudoProbe.h"
18 #include "llvm/ProfileData/SampleProfReader.h"
19 #include "llvm/Support/CommandLine.h"
20 #include "llvm/Support/Compiler.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/Error.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/LEB128.h"
25 #include "llvm/Support/ManagedStatic.h"
26 #include "llvm/Support/raw_ostream.h"
27 #include <string>
28 #include <system_error>
29 
30 using namespace llvm;
31 using namespace sampleprof;
32 
33 static cl::opt<uint64_t> ProfileSymbolListCutOff(
34     "profile-symbol-list-cutoff", cl::Hidden, cl::init(-1), cl::ZeroOrMore,
35     cl::desc("Cutoff value about how many symbols in profile symbol list "
36              "will be used. This is very useful for performance debugging"));
37 
38 namespace llvm {
39 namespace sampleprof {
40 SampleProfileFormat FunctionSamples::Format;
41 bool FunctionSamples::ProfileIsProbeBased = false;
42 bool FunctionSamples::ProfileIsCS = false;
43 bool FunctionSamples::UseMD5 = false;
44 bool FunctionSamples::HasUniqSuffix = true;
45 } // namespace sampleprof
46 } // namespace llvm
47 
48 namespace {
49 
50 // FIXME: This class is only here to support the transition to llvm::Error. It
51 // will be removed once this transition is complete. Clients should prefer to
52 // deal with the Error value directly, rather than converting to error_code.
53 class SampleProfErrorCategoryType : public std::error_category {
54   const char *name() const noexcept override { return "llvm.sampleprof"; }
55 
56   std::string message(int IE) const override {
57     sampleprof_error E = static_cast<sampleprof_error>(IE);
58     switch (E) {
59     case sampleprof_error::success:
60       return "Success";
61     case sampleprof_error::bad_magic:
62       return "Invalid sample profile data (bad magic)";
63     case sampleprof_error::unsupported_version:
64       return "Unsupported sample profile format version";
65     case sampleprof_error::too_large:
66       return "Too much profile data";
67     case sampleprof_error::truncated:
68       return "Truncated profile data";
69     case sampleprof_error::malformed:
70       return "Malformed sample profile data";
71     case sampleprof_error::unrecognized_format:
72       return "Unrecognized sample profile encoding format";
73     case sampleprof_error::unsupported_writing_format:
74       return "Profile encoding format unsupported for writing operations";
75     case sampleprof_error::truncated_name_table:
76       return "Truncated function name table";
77     case sampleprof_error::not_implemented:
78       return "Unimplemented feature";
79     case sampleprof_error::counter_overflow:
80       return "Counter overflow";
81     case sampleprof_error::ostream_seek_unsupported:
82       return "Ostream does not support seek";
83     case sampleprof_error::compress_failed:
84       return "Compress failure";
85     case sampleprof_error::uncompress_failed:
86       return "Uncompress failure";
87     case sampleprof_error::zlib_unavailable:
88       return "Zlib is unavailable";
89     case sampleprof_error::hash_mismatch:
90       return "Function hash mismatch";
91     }
92     llvm_unreachable("A value of sampleprof_error has no message.");
93   }
94 };
95 
96 } // end anonymous namespace
97 
98 static ManagedStatic<SampleProfErrorCategoryType> ErrorCategory;
99 
100 const std::error_category &llvm::sampleprof_category() {
101   return *ErrorCategory;
102 }
103 
104 void LineLocation::print(raw_ostream &OS) const {
105   OS << LineOffset;
106   if (Discriminator > 0)
107     OS << "." << Discriminator;
108 }
109 
110 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
111                                           const LineLocation &Loc) {
112   Loc.print(OS);
113   return OS;
114 }
115 
116 /// Merge the samples in \p Other into this record.
117 /// Optionally scale sample counts by \p Weight.
118 sampleprof_error SampleRecord::merge(const SampleRecord &Other,
119                                      uint64_t Weight) {
120   sampleprof_error Result;
121   // With pseudo probes, merge a dangling sample with a non-dangling sample
122   // should result in a dangling sample.
123   if (FunctionSamples::ProfileIsProbeBased &&
124       (getSamples() == FunctionSamples::InvalidProbeCount ||
125        Other.getSamples() == FunctionSamples::InvalidProbeCount)) {
126     NumSamples = FunctionSamples::InvalidProbeCount;
127     Result = sampleprof_error::success;
128   } else {
129     Result = addSamples(Other.getSamples(), Weight);
130   }
131   for (const auto &I : Other.getCallTargets()) {
132     MergeResult(Result, addCalledTarget(I.first(), I.second, Weight));
133   }
134   return Result;
135 }
136 
137 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
138 LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); }
139 #endif
140 
141 /// Print the sample record to the stream \p OS indented by \p Indent.
142 void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
143   OS << NumSamples;
144   if (hasCalls()) {
145     OS << ", calls:";
146     for (const auto &I : getSortedCallTargets())
147       OS << " " << I.first << ":" << I.second;
148   }
149   OS << "\n";
150 }
151 
152 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
153 LLVM_DUMP_METHOD void SampleRecord::dump() const { print(dbgs(), 0); }
154 #endif
155 
156 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
157                                           const SampleRecord &Sample) {
158   Sample.print(OS, 0);
159   return OS;
160 }
161 
162 /// Print the samples collected for a function on stream \p OS.
163 void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
164   if (getFunctionHash())
165     OS << "CFG checksum " << getFunctionHash() << "\n";
166 
167   OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size()
168      << " sampled lines\n";
169 
170   OS.indent(Indent);
171   if (!BodySamples.empty()) {
172     OS << "Samples collected in the function's body {\n";
173     SampleSorter<LineLocation, SampleRecord> SortedBodySamples(BodySamples);
174     for (const auto &SI : SortedBodySamples.get()) {
175       OS.indent(Indent + 2);
176       OS << SI->first << ": " << SI->second;
177     }
178     OS.indent(Indent);
179     OS << "}\n";
180   } else {
181     OS << "No samples collected in the function's body\n";
182   }
183 
184   OS.indent(Indent);
185   if (!CallsiteSamples.empty()) {
186     OS << "Samples collected in inlined callsites {\n";
187     SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
188         CallsiteSamples);
189     for (const auto &CS : SortedCallsiteSamples.get()) {
190       for (const auto &FS : CS->second) {
191         OS.indent(Indent + 2);
192         OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
193         FS.second.print(OS, Indent + 4);
194       }
195     }
196     OS.indent(Indent);
197     OS << "}\n";
198   } else {
199     OS << "No inlined callsites in this function\n";
200   }
201 }
202 
203 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
204                                           const FunctionSamples &FS) {
205   FS.print(OS);
206   return OS;
207 }
208 
209 unsigned FunctionSamples::getOffset(const DILocation *DIL) {
210   return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
211       0xffff;
212 }
213 
214 LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) {
215   if (FunctionSamples::ProfileIsProbeBased)
216     // In a pseudo-probe based profile, a callsite is simply represented by the
217     // ID of the probe associated with the call instruction. The probe ID is
218     // encoded in the Discriminator field of the call instruction's debug
219     // metadata.
220     return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex(
221                             DIL->getDiscriminator()),
222                         0);
223   else
224     return LineLocation(FunctionSamples::getOffset(DIL),
225                         DIL->getBaseDiscriminator());
226 }
227 
228 const FunctionSamples *FunctionSamples::findFunctionSamples(
229     const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
230   assert(DIL);
231   SmallVector<std::pair<LineLocation, StringRef>, 10> S;
232 
233   const DILocation *PrevDIL = DIL;
234   for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
235     S.push_back(std::make_pair(
236         LineLocation(getOffset(DIL), DIL->getBaseDiscriminator()),
237         PrevDIL->getScope()->getSubprogram()->getLinkageName()));
238     PrevDIL = DIL;
239   }
240   if (S.size() == 0)
241     return this;
242   const FunctionSamples *FS = this;
243   for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) {
244     FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper);
245   }
246   return FS;
247 }
248 
249 void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
250   NameSet.insert(Name);
251   for (const auto &BS : BodySamples)
252     for (const auto &TS : BS.second.getCallTargets())
253       NameSet.insert(TS.getKey());
254 
255   for (const auto &CS : CallsiteSamples) {
256     for (const auto &NameFS : CS.second) {
257       NameSet.insert(NameFS.first);
258       NameFS.second.findAllNames(NameSet);
259     }
260   }
261 }
262 
263 const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
264     const LineLocation &Loc, StringRef CalleeName,
265     SampleProfileReaderItaniumRemapper *Remapper) const {
266   CalleeName = getCanonicalFnName(CalleeName);
267 
268   std::string CalleeGUID;
269   CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
270 
271   auto iter = CallsiteSamples.find(Loc);
272   if (iter == CallsiteSamples.end())
273     return nullptr;
274   auto FS = iter->second.find(CalleeName);
275   if (FS != iter->second.end())
276     return &FS->second;
277   if (Remapper) {
278     if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
279       auto FS = iter->second.find(*NameInProfile);
280       if (FS != iter->second.end())
281         return &FS->second;
282     }
283   }
284   // If we cannot find exact match of the callee name, return the FS with
285   // the max total count. Only do this when CalleeName is not provided,
286   // i.e., only for indirect calls.
287   if (!CalleeName.empty())
288     return nullptr;
289   uint64_t MaxTotalSamples = 0;
290   const FunctionSamples *R = nullptr;
291   for (const auto &NameFS : iter->second)
292     if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
293       MaxTotalSamples = NameFS.second.getTotalSamples();
294       R = &NameFS.second;
295     }
296   return R;
297 }
298 
299 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
300 LLVM_DUMP_METHOD void FunctionSamples::dump() const { print(dbgs(), 0); }
301 #endif
302 
303 std::error_code ProfileSymbolList::read(const uint8_t *Data,
304                                         uint64_t ListSize) {
305   const char *ListStart = reinterpret_cast<const char *>(Data);
306   uint64_t Size = 0;
307   uint64_t StrNum = 0;
308   while (Size < ListSize && StrNum < ProfileSymbolListCutOff) {
309     StringRef Str(ListStart + Size);
310     add(Str);
311     Size += Str.size() + 1;
312     StrNum++;
313   }
314   if (Size != ListSize && StrNum != ProfileSymbolListCutOff)
315     return sampleprof_error::malformed;
316   return sampleprof_error::success;
317 }
318 
319 std::error_code ProfileSymbolList::write(raw_ostream &OS) {
320   // Sort the symbols before output. If doing compression.
321   // It will make the compression much more effective.
322   std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
323   llvm::sort(SortedList);
324 
325   std::string OutputString;
326   for (auto &Sym : SortedList) {
327     OutputString.append(Sym.str());
328     OutputString.append(1, '\0');
329   }
330 
331   OS << OutputString;
332   return sampleprof_error::success;
333 }
334 
335 void ProfileSymbolList::dump(raw_ostream &OS) const {
336   OS << "======== Dump profile symbol list ========\n";
337   std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
338   llvm::sort(SortedList);
339 
340   for (auto &Sym : SortedList)
341     OS << Sym << "\n";
342 }
343