1 //===-- ProfileGenerator.cpp - Profile Generator  ---------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ProfileGenerator.h"
10 
11 static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
12                                            cl::Required,
13                                            cl::desc("Output profile file"));
14 
15 static cl::opt<SampleProfileFormat> OutputFormat(
16     "format", cl::desc("Format of output profile"), cl::init(SPF_Text),
17     cl::values(
18         clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
19         clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"),
20         clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
21         clEnumValN(SPF_Text, "text", "Text encoding"),
22         clEnumValN(SPF_GCC, "gcc",
23                    "GCC encoding (only meaningful for -sample)")));
24 
25 using namespace llvm;
26 using namespace sampleprof;
27 
28 namespace llvm {
29 namespace sampleprof {
30 
31 static bool
32 usePseudoProbes(const BinarySampleCounterMap &BinarySampleCounters) {
33   return BinarySampleCounters.size() &&
34          BinarySampleCounters.begin()->first->usePseudoProbes();
35 }
36 
37 std::unique_ptr<ProfileGenerator>
38 ProfileGenerator::create(const BinarySampleCounterMap &BinarySampleCounters,
39                          enum PerfScriptType SampleType) {
40   std::unique_ptr<ProfileGenerator> ProfileGenerator;
41   if (SampleType == PERF_LBR_STACK) {
42     if (usePseudoProbes(BinarySampleCounters)) {
43       ProfileGenerator.reset(
44           new PseudoProbeCSProfileGenerator(BinarySampleCounters));
45     } else {
46       ProfileGenerator.reset(new CSProfileGenerator(BinarySampleCounters));
47     }
48   } else {
49     // TODO:
50     llvm_unreachable("Unsupported perfscript!");
51   }
52 
53   return ProfileGenerator;
54 }
55 
56 void ProfileGenerator::write() {
57   auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat);
58   if (std::error_code EC = WriterOrErr.getError())
59     exitWithError(EC, OutputFilename);
60   auto Writer = std::move(WriterOrErr.get());
61   Writer->write(ProfileMap);
62 }
63 
64 void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges,
65                                           const RangeSample &Ranges) {
66 
67   /*
68   Regions may overlap with each other. Using the boundary info, find all
69   disjoint ranges and their sample count. BoundaryPoint contains the count
70   mutiple samples begin/end at this points.
71 
72   |<--100-->|           Sample1
73   |<------200------>|   Sample2
74   A         B       C
75 
76   In the example above,
77   Sample1 begins at A, ends at B, its value is 100.
78   Sample2 beings at A, ends at C, its value is 200.
79   For A, BeginCount is the sum of sample begins at A, which is 300 and no
80   samples ends at A, so EndCount is 0.
81   Then boundary points A, B, and C with begin/end counts are:
82   A: (300, 0)
83   B: (0, 100)
84   C: (0, 200)
85   */
86   struct BoundaryPoint {
87     // Sum of sample counts beginning at this point
88     uint64_t BeginCount;
89     // Sum of sample counts ending at this point
90     uint64_t EndCount;
91 
92     BoundaryPoint() : BeginCount(0), EndCount(0){};
93 
94     void addBeginCount(uint64_t Count) { BeginCount += Count; }
95 
96     void addEndCount(uint64_t Count) { EndCount += Count; }
97   };
98 
99   /*
100   For the above example. With boundary points, follwing logic finds two
101   disjoint region of
102 
103   [A,B]:   300
104   [B+1,C]: 200
105 
106   If there is a boundary point that both begin and end, the point itself
107   becomes a separate disjoint region. For example, if we have original
108   ranges of
109 
110   |<--- 100 --->|
111                 |<--- 200 --->|
112   A             B             C
113 
114   there are three boundary points with their begin/end counts of
115 
116   A: (100, 0)
117   B: (200, 100)
118   C: (0, 200)
119 
120   the disjoint ranges would be
121 
122   [A, B-1]: 100
123   [B, B]:   300
124   [B+1, C]: 200.
125   */
126   std::map<uint64_t, BoundaryPoint> Boundaries;
127 
128   for (auto Item : Ranges) {
129     uint64_t Begin = Item.first.first;
130     uint64_t End = Item.first.second;
131     uint64_t Count = Item.second;
132     if (Boundaries.find(Begin) == Boundaries.end())
133       Boundaries[Begin] = BoundaryPoint();
134     Boundaries[Begin].addBeginCount(Count);
135 
136     if (Boundaries.find(End) == Boundaries.end())
137       Boundaries[End] = BoundaryPoint();
138     Boundaries[End].addEndCount(Count);
139   }
140 
141   uint64_t BeginAddress = 0;
142   int Count = 0;
143   for (auto Item : Boundaries) {
144     uint64_t Address = Item.first;
145     BoundaryPoint &Point = Item.second;
146     if (Point.BeginCount) {
147       if (BeginAddress)
148         DisjointRanges[{BeginAddress, Address - 1}] = Count;
149       Count += Point.BeginCount;
150       BeginAddress = Address;
151     }
152     if (Point.EndCount) {
153       assert(BeginAddress && "First boundary point cannot be 'end' point");
154       DisjointRanges[{BeginAddress, Address}] = Count;
155       Count -= Point.EndCount;
156       BeginAddress = Address + 1;
157     }
158   }
159 }
160 
161 FunctionSamples &
162 CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
163   auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
164   if (Ret.second) {
165     SampleContext FContext(Ret.first->first(), RawContext);
166     FunctionSamples &FProfile = Ret.first->second;
167     FProfile.setName(FContext.getName());
168     FProfile.setContext(FContext);
169   }
170   return Ret.first->second;
171 }
172 
173 void CSProfileGenerator::updateBodySamplesforFunctionProfile(
174     FunctionSamples &FunctionProfile, const FrameLocation &LeafLoc,
175     uint64_t Count) {
176   // Filter out invalid negative(int type) lineOffset
177   if (LeafLoc.second.LineOffset & 0x80000000)
178     return;
179   // Use the maximum count of samples with same line location
180   ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt(
181       LeafLoc.second.LineOffset, LeafLoc.second.Discriminator);
182   uint64_t PreviousCount = R ? R.get() : 0;
183   if (PreviousCount < Count) {
184     FunctionProfile.addBodySamples(LeafLoc.second.LineOffset,
185                                    LeafLoc.second.Discriminator,
186                                    Count - PreviousCount);
187     FunctionProfile.addTotalSamples(Count - PreviousCount);
188   }
189 }
190 
191 void CSProfileGenerator::populateFunctionBodySamples(
192     FunctionSamples &FunctionProfile, const RangeSample &RangeCounter,
193     ProfiledBinary *Binary) {
194   // Compute disjoint ranges first, so we can use MAX
195   // for calculating count for each location.
196   RangeSample Ranges;
197   findDisjointRanges(Ranges, RangeCounter);
198   for (auto Range : Ranges) {
199     uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first);
200     uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second);
201     uint64_t Count = Range.second;
202     // Disjoint ranges have introduce zero-filled gap that
203     // doesn't belong to current context, filter them out.
204     if (Count == 0)
205       continue;
206 
207     InstructionPointer IP(Binary, RangeBegin, true);
208 
209     // Disjoint ranges may have range in the middle of two instr,
210     // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range
211     // can be Addr1+1 to Addr2-1. We should ignore such range.
212     if (IP.Address > RangeEnd)
213       continue;
214 
215     while (IP.Address <= RangeEnd) {
216       uint64_t Offset = Binary->virtualAddrToOffset(IP.Address);
217       const FrameLocation &LeafLoc = Binary->getInlineLeafFrameLoc(Offset);
218       // Recording body sample for this specific context
219       updateBodySamplesforFunctionProfile(FunctionProfile, LeafLoc, Count);
220       // Move to next IP within the range
221       IP.advance();
222     }
223   }
224 }
225 
226 void CSProfileGenerator::populateFunctionBoundarySamples(
227     StringRef ContextId, FunctionSamples &FunctionProfile,
228     const BranchSample &BranchCounters, ProfiledBinary *Binary) {
229 
230   for (auto Entry : BranchCounters) {
231     uint64_t SourceOffset = Entry.first.first;
232     uint64_t TargetOffset = Entry.first.second;
233     uint64_t Count = Entry.second;
234     // Get the callee name by branch target if it's a call branch
235     StringRef CalleeName = FunctionSamples::getCanonicalFnName(
236         Binary->getFuncFromStartOffset(TargetOffset));
237     if (CalleeName.size() == 0)
238       continue;
239 
240     // Record called target sample and its count
241     const FrameLocation &LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset);
242     FunctionProfile.addCalledTargetSamples(LeafLoc.second.LineOffset,
243                                            LeafLoc.second.Discriminator,
244                                            CalleeName, Count);
245 
246     // Record head sample for called target(callee)
247     // TODO: Cleanup ' @ '
248     std::string CalleeContextId =
249         getCallSite(LeafLoc) + " @ " + CalleeName.str();
250     if (ContextId.find(" @ ") != StringRef::npos) {
251       CalleeContextId =
252           ContextId.rsplit(" @ ").first.str() + " @ " + CalleeContextId;
253     }
254 
255     FunctionSamples &CalleeProfile =
256         getFunctionProfileForContext(CalleeContextId);
257     assert(Count != 0 && "Unexpected zero weight branch");
258     CalleeProfile.addHeadSamples(Count);
259   }
260 }
261 
262 static FrameLocation getCallerContext(StringRef CalleeContext,
263                                       StringRef &CallerNameWithContext) {
264   StringRef CallerContext = CalleeContext.rsplit(" @ ").first;
265   CallerNameWithContext = CallerContext.rsplit(':').first;
266   auto ContextSplit = CallerContext.rsplit(" @ ");
267   FrameLocation LeafFrameLoc = {"", {0, 0}};
268   StringRef Funcname;
269   SampleContext::decodeContextString(ContextSplit.second, Funcname,
270                                      LeafFrameLoc.second);
271   LeafFrameLoc.first = Funcname.str();
272   return LeafFrameLoc;
273 }
274 
275 void CSProfileGenerator::populateInferredFunctionSamples() {
276   for (const auto &Item : ProfileMap) {
277     const StringRef CalleeContext = Item.first();
278     const FunctionSamples &CalleeProfile = Item.second;
279 
280     // If we already have head sample counts, we must have value profile
281     // for call sites added already. Skip to avoid double counting.
282     if (CalleeProfile.getHeadSamples())
283       continue;
284     // If we don't have context, nothing to do for caller's call site.
285     // This could happen for entry point function.
286     if (CalleeContext.find(" @ ") == StringRef::npos)
287       continue;
288 
289     // Infer Caller's frame loc and context ID through string splitting
290     StringRef CallerContextId;
291     FrameLocation &&CallerLeafFrameLoc =
292         getCallerContext(CalleeContext, CallerContextId);
293 
294     // It's possible that we haven't seen any sample directly in the caller,
295     // in which case CallerProfile will not exist. But we can't modify
296     // ProfileMap while iterating it.
297     // TODO: created function profile for those callers too
298     if (ProfileMap.find(CallerContextId) == ProfileMap.end())
299       continue;
300     FunctionSamples &CallerProfile = ProfileMap[CallerContextId];
301 
302     // Since we don't have call count for inlined functions, we
303     // estimate it from inlinee's profile using entry body sample.
304     uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples();
305     // If we don't have samples with location, use 1 to indicate live.
306     if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size())
307       EstimatedCallCount = 1;
308     CallerProfile.addCalledTargetSamples(
309         CallerLeafFrameLoc.second.LineOffset,
310         CallerLeafFrameLoc.second.Discriminator, CalleeProfile.getName(),
311         EstimatedCallCount);
312     CallerProfile.addBodySamples(CallerLeafFrameLoc.second.LineOffset,
313                                  CallerLeafFrameLoc.second.Discriminator,
314                                  EstimatedCallCount);
315     CallerProfile.addTotalSamples(EstimatedCallCount);
316   }
317 }
318 
319 } // end namespace sampleprof
320 } // end namespace llvm
321