1 //===-- ProfileGenerator.cpp - Profile Generator  ---------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ProfileGenerator.h"
10 
11 static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
12                                            cl::Required,
13                                            cl::desc("Output profile file"));
14 
15 static cl::opt<SampleProfileFormat> OutputFormat(
16     "format", cl::desc("Format of output profile"), cl::init(SPF_Text),
17     cl::values(
18         clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
19         clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"),
20         clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
21         clEnumValN(SPF_Text, "text", "Text encoding"),
22         clEnumValN(SPF_GCC, "gcc",
23                    "GCC encoding (only meaningful for -sample)")));
24 
25 using namespace llvm;
26 using namespace sampleprof;
27 
28 namespace llvm {
29 namespace sampleprof {
30 
31 static bool
32 usePseudoProbes(const BinarySampleCounterMap &BinarySampleCounters) {
33   return BinarySampleCounters.size() &&
34          BinarySampleCounters.begin()->first->usePseudoProbes();
35 }
36 
37 std::unique_ptr<ProfileGenerator>
38 ProfileGenerator::create(const BinarySampleCounterMap &BinarySampleCounters,
39                          enum PerfScriptType SampleType) {
40   std::unique_ptr<ProfileGenerator> ProfileGenerator;
41   if (SampleType == PERF_LBR_STACK) {
42     if (usePseudoProbes(BinarySampleCounters)) {
43       ProfileGenerator.reset(
44           new PseudoProbeCSProfileGenerator(BinarySampleCounters));
45     } else {
46       ProfileGenerator.reset(new CSProfileGenerator(BinarySampleCounters));
47     }
48   } else {
49     // TODO:
50     llvm_unreachable("Unsupported perfscript!");
51   }
52 
53   return ProfileGenerator;
54 }
55 
56 void ProfileGenerator::write() {
57   auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat);
58   if (std::error_code EC = WriterOrErr.getError())
59     exitWithError(EC, OutputFilename);
60   auto Writer = std::move(WriterOrErr.get());
61   Writer->write(ProfileMap);
62 }
63 
64 void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges,
65                                           const RangeSample &Ranges) {
66 
67   /*
68   Regions may overlap with each other. Using the boundary info, find all
69   disjoint ranges and their sample count. BoundaryPoint contains the count
70   mutiple samples begin/end at this points.
71 
72   |<--100-->|           Sample1
73   |<------200------>|   Sample2
74   A         B       C
75 
76   In the example above,
77   Sample1 begins at A, ends at B, its value is 100.
78   Sample2 beings at A, ends at C, its value is 200.
79   For A, BeginCount is the sum of sample begins at A, which is 300 and no
80   samples ends at A, so EndCount is 0.
81   Then boundary points A, B, and C with begin/end counts are:
82   A: (300, 0)
83   B: (0, 100)
84   C: (0, 200)
85   */
86   struct BoundaryPoint {
87     // Sum of sample counts beginning at this point
88     uint64_t BeginCount;
89     // Sum of sample counts ending at this point
90     uint64_t EndCount;
91 
92     BoundaryPoint() : BeginCount(0), EndCount(0){};
93 
94     void addBeginCount(uint64_t Count) { BeginCount += Count; }
95 
96     void addEndCount(uint64_t Count) { EndCount += Count; }
97   };
98 
99   /*
100   For the above example. With boundary points, follwing logic finds two
101   disjoint region of
102 
103   [A,B]:   300
104   [B+1,C]: 200
105 
106   If there is a boundary point that both begin and end, the point itself
107   becomes a separate disjoint region. For example, if we have original
108   ranges of
109 
110   |<--- 100 --->|
111                 |<--- 200 --->|
112   A             B             C
113 
114   there are three boundary points with their begin/end counts of
115 
116   A: (100, 0)
117   B: (200, 100)
118   C: (0, 200)
119 
120   the disjoint ranges would be
121 
122   [A, B-1]: 100
123   [B, B]:   300
124   [B+1, C]: 200.
125   */
126   std::map<uint64_t, BoundaryPoint> Boundaries;
127 
128   for (auto Item : Ranges) {
129     uint64_t Begin = Item.first.first;
130     uint64_t End = Item.first.second;
131     uint64_t Count = Item.second;
132     if (Boundaries.find(Begin) == Boundaries.end())
133       Boundaries[Begin] = BoundaryPoint();
134     Boundaries[Begin].addBeginCount(Count);
135 
136     if (Boundaries.find(End) == Boundaries.end())
137       Boundaries[End] = BoundaryPoint();
138     Boundaries[End].addEndCount(Count);
139   }
140 
141   uint64_t BeginAddress = 0;
142   int Count = 0;
143   for (auto Item : Boundaries) {
144     uint64_t Address = Item.first;
145     BoundaryPoint &Point = Item.second;
146     if (Point.BeginCount) {
147       if (BeginAddress)
148         DisjointRanges[{BeginAddress, Address - 1}] = Count;
149       Count += Point.BeginCount;
150       BeginAddress = Address;
151     }
152     if (Point.EndCount) {
153       assert(BeginAddress && "First boundary point cannot be 'end' point");
154       DisjointRanges[{BeginAddress, Address}] = Count;
155       Count -= Point.EndCount;
156       BeginAddress = Address + 1;
157     }
158   }
159 }
160 
161 FunctionSamples &
162 CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
163   auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
164   if (Ret.second) {
165     SampleContext FContext(Ret.first->first(), RawContext);
166     FunctionSamples &FProfile = Ret.first->second;
167     FProfile.setName(FContext.getName());
168     FProfile.setContext(FContext);
169   }
170   return Ret.first->second;
171 }
172 
173 void CSProfileGenerator::updateBodySamplesforFunctionProfile(
174     FunctionSamples &FunctionProfile, const FrameLocation &LeafLoc,
175     uint64_t Count) {
176   // Filter out invalid negative(int type) lineOffset
177   if (LeafLoc.second.LineOffset & 0x80000000)
178     return;
179   // Use the maximum count of samples with same line location
180   ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt(
181       LeafLoc.second.LineOffset, LeafLoc.second.Discriminator);
182   uint64_t PreviousCount = R ? R.get() : 0;
183   if (PreviousCount < Count) {
184     FunctionProfile.addBodySamples(LeafLoc.second.LineOffset,
185                                    LeafLoc.second.Discriminator,
186                                    Count - PreviousCount);
187     FunctionProfile.addTotalSamples(Count - PreviousCount);
188   }
189 }
190 
191 void CSProfileGenerator::populateFunctionBodySamples(
192     FunctionSamples &FunctionProfile, const RangeSample &RangeCounter,
193     ProfiledBinary *Binary) {
194   // Compute disjoint ranges first, so we can use MAX
195   // for calculating count for each location.
196   RangeSample Ranges;
197   findDisjointRanges(Ranges, RangeCounter);
198   for (auto Range : Ranges) {
199     uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first);
200     uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second);
201     uint64_t Count = Range.second;
202     // Disjoint ranges have introduce zero-filled gap that
203     // doesn't belong to current context, filter them out.
204     if (Count == 0)
205       continue;
206 
207     InstructionPointer IP(Binary, RangeBegin, true);
208 
209     // Disjoint ranges may have range in the middle of two instr,
210     // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range
211     // can be Addr1+1 to Addr2-1. We should ignore such range.
212     if (IP.Address > RangeEnd)
213       continue;
214 
215     while (IP.Address <= RangeEnd) {
216       uint64_t Offset = Binary->virtualAddrToOffset(IP.Address);
217       const FrameLocation &LeafLoc = Binary->getInlineLeafFrameLoc(Offset);
218       // Recording body sample for this specific context
219       updateBodySamplesforFunctionProfile(FunctionProfile, LeafLoc, Count);
220       // Move to next IP within the range
221       IP.advance();
222     }
223   }
224 }
225 
226 void CSProfileGenerator::populateFunctionBoundarySamples(
227     StringRef ContextId, FunctionSamples &FunctionProfile,
228     const BranchSample &BranchCounters, ProfiledBinary *Binary) {
229 
230   for (auto Entry : BranchCounters) {
231     uint64_t SourceOffset = Entry.first.first;
232     uint64_t TargetOffset = Entry.first.second;
233     uint64_t Count = Entry.second;
234     // Get the callee name by branch target if it's a call branch
235     StringRef CalleeName = FunctionSamples::getCanonicalFnName(
236         Binary->getFuncFromStartOffset(TargetOffset));
237     if (CalleeName.size() == 0)
238       continue;
239 
240     // Record called target sample and its count
241     const FrameLocation &LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset);
242 
243     FunctionProfile.addCalledTargetSamples(LeafLoc.second.LineOffset,
244                                            LeafLoc.second.Discriminator,
245                                            CalleeName, Count);
246     FunctionProfile.addTotalSamples(Count);
247 
248     // Record head sample for called target(callee)
249     // TODO: Cleanup ' @ '
250     std::string CalleeContextId =
251         getCallSite(LeafLoc) + " @ " + CalleeName.str();
252     if (ContextId.find(" @ ") != StringRef::npos) {
253       CalleeContextId =
254           ContextId.rsplit(" @ ").first.str() + " @ " + CalleeContextId;
255     }
256 
257     FunctionSamples &CalleeProfile =
258         getFunctionProfileForContext(CalleeContextId);
259     assert(Count != 0 && "Unexpected zero weight branch");
260     CalleeProfile.addHeadSamples(Count);
261   }
262 }
263 
264 static FrameLocation getCallerContext(StringRef CalleeContext,
265                                       StringRef &CallerNameWithContext) {
266   StringRef CallerContext = CalleeContext.rsplit(" @ ").first;
267   CallerNameWithContext = CallerContext.rsplit(':').first;
268   auto ContextSplit = CallerContext.rsplit(" @ ");
269   FrameLocation LeafFrameLoc = {"", {0, 0}};
270   StringRef Funcname;
271   SampleContext::decodeContextString(ContextSplit.second, Funcname,
272                                      LeafFrameLoc.second);
273   LeafFrameLoc.first = Funcname.str();
274   return LeafFrameLoc;
275 }
276 
277 void CSProfileGenerator::populateInferredFunctionSamples() {
278   for (const auto &Item : ProfileMap) {
279     const StringRef CalleeContext = Item.first();
280     const FunctionSamples &CalleeProfile = Item.second;
281 
282     // If we already have head sample counts, we must have value profile
283     // for call sites added already. Skip to avoid double counting.
284     if (CalleeProfile.getHeadSamples())
285       continue;
286     // If we don't have context, nothing to do for caller's call site.
287     // This could happen for entry point function.
288     if (CalleeContext.find(" @ ") == StringRef::npos)
289       continue;
290 
291     // Infer Caller's frame loc and context ID through string splitting
292     StringRef CallerContextId;
293     FrameLocation &&CallerLeafFrameLoc =
294         getCallerContext(CalleeContext, CallerContextId);
295 
296     // It's possible that we haven't seen any sample directly in the caller,
297     // in which case CallerProfile will not exist. But we can't modify
298     // ProfileMap while iterating it.
299     // TODO: created function profile for those callers too
300     if (ProfileMap.find(CallerContextId) == ProfileMap.end())
301       continue;
302     FunctionSamples &CallerProfile = ProfileMap[CallerContextId];
303 
304     // Since we don't have call count for inlined functions, we
305     // estimate it from inlinee's profile using entry body sample.
306     uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples();
307     // If we don't have samples with location, use 1 to indicate live.
308     if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size())
309       EstimatedCallCount = 1;
310     CallerProfile.addCalledTargetSamples(
311         CallerLeafFrameLoc.second.LineOffset,
312         CallerLeafFrameLoc.second.Discriminator, CalleeProfile.getName(),
313         EstimatedCallCount);
314     updateBodySamplesforFunctionProfile(CallerProfile, CallerLeafFrameLoc,
315                                         EstimatedCallCount);
316   }
317 }
318 
319 } // end namespace sampleprof
320 } // end namespace llvm
321