1 //===-- ProfileGenerator.cpp - Profile Generator  ---------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ProfileGenerator.h"
10 #include "ProfiledBinary.h"
11 #include "llvm/ProfileData/ProfileCommon.h"
12 #include <unordered_set>
13 
14 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
15                                     cl::Required,
16                                     cl::desc("Output profile file"));
17 static cl::alias OutputA("o", cl::desc("Alias for --output"),
18                          cl::aliasopt(OutputFilename));
19 
20 static cl::opt<SampleProfileFormat> OutputFormat(
21     "format", cl::desc("Format of output profile"), cl::init(SPF_Ext_Binary),
22     cl::values(
23         clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
24         clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"),
25         clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
26         clEnumValN(SPF_Text, "text", "Text encoding"),
27         clEnumValN(SPF_GCC, "gcc",
28                    "GCC encoding (only meaningful for -sample)")));
29 
30 cl::opt<bool> UseMD5(
31     "use-md5", cl::init(false), cl::Hidden,
32     cl::desc("Use md5 to represent function names in the output profile (only "
33              "meaningful for -extbinary)"));
34 
35 static cl::opt<int32_t, true> RecursionCompression(
36     "compress-recursion",
37     cl::desc("Compressing recursion by deduplicating adjacent frame "
38              "sequences up to the specified size. -1 means no size limit."),
39     cl::Hidden,
40     cl::location(llvm::sampleprof::CSProfileGenerator::MaxCompressionSize));
41 
42 static cl::opt<bool> CSProfMergeColdContext(
43     "csprof-merge-cold-context", cl::init(true), cl::ZeroOrMore,
44     cl::desc("If the total count of context profile is smaller than "
45              "the threshold, it will be merged into context-less base "
46              "profile."));
47 
48 static cl::opt<bool> CSProfTrimColdContext(
49     "csprof-trim-cold-context", cl::init(false), cl::ZeroOrMore,
50     cl::desc("If the total count of the profile after all merge is done "
51              "is still smaller than threshold, it will be trimmed."));
52 
53 static cl::opt<uint32_t> CSProfMaxColdContextDepth(
54     "csprof-max-cold-context-depth", cl::init(1), cl::ZeroOrMore,
55     cl::desc("Keep the last K contexts while merging cold profile. 1 means the "
56              "context-less base profile"));
57 
58 static cl::opt<int, true> CSProfMaxContextDepth(
59     "csprof-max-context-depth", cl::ZeroOrMore,
60     cl::desc("Keep the last K contexts while merging profile. -1 means no "
61              "depth limit."),
62     cl::location(llvm::sampleprof::CSProfileGenerator::MaxContextDepth));
63 
64 extern cl::opt<int> ProfileSummaryCutoffCold;
65 
66 using namespace llvm;
67 using namespace sampleprof;
68 
69 namespace llvm {
70 namespace sampleprof {
71 
72 // Initialize the MaxCompressionSize to -1 which means no size limit
73 int32_t CSProfileGenerator::MaxCompressionSize = -1;
74 
75 int CSProfileGenerator::MaxContextDepth = -1;
76 
77 std::unique_ptr<ProfileGenerator>
78 ProfileGenerator::create(ProfiledBinary *Binary,
79                          const ContextSampleCounterMap &SampleCounters,
80                          enum PerfScriptType SampleType) {
81   std::unique_ptr<ProfileGenerator> ProfileGenerator;
82   if (SampleType == PERF_LBR_STACK) {
83     if (Binary->usePseudoProbes()) {
84       ProfileGenerator.reset(
85           new PseudoProbeCSProfileGenerator(Binary, SampleCounters));
86     } else {
87       ProfileGenerator.reset(new CSProfileGenerator(Binary, SampleCounters));
88     }
89   } else {
90     // TODO:
91     llvm_unreachable("Unsupported perfscript!");
92   }
93 
94   return ProfileGenerator;
95 }
96 
97 void ProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer,
98                              SampleProfileMap &ProfileMap) {
99   if (std::error_code EC = Writer->write(ProfileMap))
100     exitWithError(std::move(EC));
101 }
102 
103 void ProfileGenerator::write() {
104   auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat);
105   if (std::error_code EC = WriterOrErr.getError())
106     exitWithError(EC, OutputFilename);
107 
108   if (UseMD5) {
109     if (OutputFormat != SPF_Ext_Binary)
110       WithColor::warning() << "-use-md5 is ignored. Specify "
111                               "--format=extbinary to enable it\n";
112     else
113       WriterOrErr.get()->setUseMD5();
114   }
115 
116   write(std::move(WriterOrErr.get()), ProfileMap);
117 }
118 
119 void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges,
120                                           const RangeSample &Ranges) {
121 
122   /*
123   Regions may overlap with each other. Using the boundary info, find all
124   disjoint ranges and their sample count. BoundaryPoint contains the count
125   multiple samples begin/end at this points.
126 
127   |<--100-->|           Sample1
128   |<------200------>|   Sample2
129   A         B       C
130 
131   In the example above,
132   Sample1 begins at A, ends at B, its value is 100.
133   Sample2 beings at A, ends at C, its value is 200.
134   For A, BeginCount is the sum of sample begins at A, which is 300 and no
135   samples ends at A, so EndCount is 0.
136   Then boundary points A, B, and C with begin/end counts are:
137   A: (300, 0)
138   B: (0, 100)
139   C: (0, 200)
140   */
141   struct BoundaryPoint {
142     // Sum of sample counts beginning at this point
143     uint64_t BeginCount;
144     // Sum of sample counts ending at this point
145     uint64_t EndCount;
146 
147     BoundaryPoint() : BeginCount(0), EndCount(0){};
148 
149     void addBeginCount(uint64_t Count) { BeginCount += Count; }
150 
151     void addEndCount(uint64_t Count) { EndCount += Count; }
152   };
153 
154   /*
155   For the above example. With boundary points, follwing logic finds two
156   disjoint region of
157 
158   [A,B]:   300
159   [B+1,C]: 200
160 
161   If there is a boundary point that both begin and end, the point itself
162   becomes a separate disjoint region. For example, if we have original
163   ranges of
164 
165   |<--- 100 --->|
166                 |<--- 200 --->|
167   A             B             C
168 
169   there are three boundary points with their begin/end counts of
170 
171   A: (100, 0)
172   B: (200, 100)
173   C: (0, 200)
174 
175   the disjoint ranges would be
176 
177   [A, B-1]: 100
178   [B, B]:   300
179   [B+1, C]: 200.
180   */
181   std::map<uint64_t, BoundaryPoint> Boundaries;
182 
183   for (auto Item : Ranges) {
184     uint64_t Begin = Item.first.first;
185     uint64_t End = Item.first.second;
186     uint64_t Count = Item.second;
187     if (Boundaries.find(Begin) == Boundaries.end())
188       Boundaries[Begin] = BoundaryPoint();
189     Boundaries[Begin].addBeginCount(Count);
190 
191     if (Boundaries.find(End) == Boundaries.end())
192       Boundaries[End] = BoundaryPoint();
193     Boundaries[End].addEndCount(Count);
194   }
195 
196   uint64_t BeginAddress = UINT64_MAX;
197   int Count = 0;
198   for (auto Item : Boundaries) {
199     uint64_t Address = Item.first;
200     BoundaryPoint &Point = Item.second;
201     if (Point.BeginCount) {
202       if (BeginAddress != UINT64_MAX)
203         DisjointRanges[{BeginAddress, Address - 1}] = Count;
204       Count += Point.BeginCount;
205       BeginAddress = Address;
206     }
207     if (Point.EndCount) {
208       assert((BeginAddress != UINT64_MAX) &&
209              "First boundary point cannot be 'end' point");
210       DisjointRanges[{BeginAddress, Address}] = Count;
211       Count -= Point.EndCount;
212       BeginAddress = Address + 1;
213     }
214   }
215 }
216 
217 FunctionSamples &
218 CSProfileGenerator::getFunctionProfileForContext(SampleContextFrames Context,
219                                                  bool WasLeafInlined) {
220   SampleContext FContext(Context);
221   auto Ret = ProfileMap.emplace(Context, FunctionSamples());
222   if (Ret.second) {
223     SampleContext FContext(Context, RawContext);
224     if (WasLeafInlined)
225       FContext.setAttribute(ContextWasInlined);
226     FunctionSamples &FProfile = Ret.first->second;
227     FProfile.setContext(FContext);
228   }
229   return Ret.first->second;
230 }
231 
232 void CSProfileGenerator::generateProfile() {
233   FunctionSamples::ProfileIsCS = true;
234   for (const auto &CI : SampleCounters) {
235     const StringBasedCtxKey *CtxKey =
236         dyn_cast<StringBasedCtxKey>(CI.first.getPtr());
237     // Get or create function profile for the range
238     FunctionSamples &FunctionProfile =
239         getFunctionProfileForContext(CtxKey->Context, CtxKey->WasLeafInlined);
240 
241     // Fill in function body samples
242     populateFunctionBodySamples(FunctionProfile, CI.second.RangeCounter);
243     // Fill in boundary sample counts as well as call site samples for calls
244     populateFunctionBoundarySamples(CtxKey->Context, FunctionProfile,
245                                     CI.second.BranchCounter);
246   }
247   // Fill in call site value sample for inlined calls and also use context to
248   // infer missing samples. Since we don't have call count for inlined
249   // functions, we estimate it from inlinee's profile using the entry of the
250   // body sample.
251   populateInferredFunctionSamples();
252 
253   postProcessProfiles();
254 }
255 
256 void CSProfileGenerator::updateBodySamplesforFunctionProfile(
257     FunctionSamples &FunctionProfile, const SampleContextFrame &LeafLoc,
258     uint64_t Count) {
259   // Filter out invalid negative(int type) lineOffset
260   if (LeafLoc.Callsite.LineOffset & 0x80000000)
261     return;
262   // Use the maximum count of samples with same line location
263   ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt(
264       LeafLoc.Callsite.LineOffset, LeafLoc.Callsite.Discriminator);
265   uint64_t PreviousCount = R ? R.get() : 0;
266   if (PreviousCount < Count) {
267     FunctionProfile.addBodySamples(LeafLoc.Callsite.LineOffset,
268                                    LeafLoc.Callsite.Discriminator,
269                                    Count - PreviousCount);
270   }
271 }
272 
273 void CSProfileGenerator::populateFunctionBodySamples(
274     FunctionSamples &FunctionProfile, const RangeSample &RangeCounter) {
275   // Compute disjoint ranges first, so we can use MAX
276   // for calculating count for each location.
277   RangeSample Ranges;
278   findDisjointRanges(Ranges, RangeCounter);
279   for (auto Range : Ranges) {
280     uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first);
281     uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second);
282     uint64_t Count = Range.second;
283     // Disjoint ranges have introduce zero-filled gap that
284     // doesn't belong to current context, filter them out.
285     if (Count == 0)
286       continue;
287 
288     InstructionPointer IP(Binary, RangeBegin, true);
289 
290     // Disjoint ranges may have range in the middle of two instr,
291     // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range
292     // can be Addr1+1 to Addr2-1. We should ignore such range.
293     if (IP.Address > RangeEnd)
294       continue;
295 
296     while (IP.Address <= RangeEnd) {
297       uint64_t Offset = Binary->virtualAddrToOffset(IP.Address);
298       auto LeafLoc = Binary->getInlineLeafFrameLoc(Offset);
299       if (LeafLoc.hasValue()) {
300         // Recording body sample for this specific context
301         updateBodySamplesforFunctionProfile(FunctionProfile, *LeafLoc, Count);
302       }
303       // Accumulate total sample count even it's a line with invalid debug info
304       FunctionProfile.addTotalSamples(Count);
305       // Move to next IP within the range
306       IP.advance();
307     }
308   }
309 }
310 
311 void CSProfileGenerator::populateFunctionBoundarySamples(
312     SampleContextFrames ContextId, FunctionSamples &FunctionProfile,
313     const BranchSample &BranchCounters) {
314 
315   for (auto Entry : BranchCounters) {
316     uint64_t SourceOffset = Entry.first.first;
317     uint64_t TargetOffset = Entry.first.second;
318     uint64_t Count = Entry.second;
319     // Get the callee name by branch target if it's a call branch
320     StringRef CalleeName = FunctionSamples::getCanonicalFnName(
321         Binary->getFuncFromStartOffset(TargetOffset));
322     if (CalleeName.size() == 0)
323       continue;
324 
325     // Record called target sample and its count
326     auto LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset);
327     if (!LeafLoc.hasValue())
328       continue;
329     FunctionProfile.addCalledTargetSamples(LeafLoc->Callsite.LineOffset,
330                                            LeafLoc->Callsite.Discriminator,
331                                            CalleeName, Count);
332 
333     // Record head sample for called target(callee)
334     SampleContextFrameVector CalleeCtx(ContextId.begin(), ContextId.end());
335     assert(CalleeCtx.back().CallerName == LeafLoc->CallerName &&
336            "Leaf function name doesn't match");
337     CalleeCtx.back() = *LeafLoc;
338     CalleeCtx.emplace_back(CalleeName, LineLocation(0, 0));
339     FunctionSamples &CalleeProfile = getFunctionProfileForContext(CalleeCtx);
340     assert(Count != 0 && "Unexpected zero weight branch");
341     CalleeProfile.addHeadSamples(Count);
342   }
343 }
344 
345 static SampleContextFrame
346 getCallerContext(SampleContextFrames CalleeContext,
347                  SampleContextFrameVector &CallerContext) {
348   assert(CalleeContext.size() > 1 && "Unexpected empty context");
349   CalleeContext = CalleeContext.drop_back();
350   CallerContext.assign(CalleeContext.begin(), CalleeContext.end());
351   SampleContextFrame CallerFrame = CallerContext.back();
352   CallerContext.back().Callsite = LineLocation(0, 0);
353   return CallerFrame;
354 }
355 
356 void CSProfileGenerator::populateInferredFunctionSamples() {
357   for (const auto &Item : ProfileMap) {
358     const auto &CalleeContext = Item.first;
359     const FunctionSamples &CalleeProfile = Item.second;
360 
361     // If we already have head sample counts, we must have value profile
362     // for call sites added already. Skip to avoid double counting.
363     if (CalleeProfile.getHeadSamples())
364       continue;
365     // If we don't have context, nothing to do for caller's call site.
366     // This could happen for entry point function.
367     if (CalleeContext.isBaseContext())
368       continue;
369 
370     // Infer Caller's frame loc and context ID through string splitting
371     SampleContextFrameVector CallerContextId;
372     SampleContextFrame &&CallerLeafFrameLoc =
373         getCallerContext(CalleeContext.getContextFrames(), CallerContextId);
374     SampleContextFrames CallerContext(CallerContextId);
375 
376     // It's possible that we haven't seen any sample directly in the caller,
377     // in which case CallerProfile will not exist. But we can't modify
378     // ProfileMap while iterating it.
379     // TODO: created function profile for those callers too
380     if (ProfileMap.find(CallerContext) == ProfileMap.end())
381       continue;
382     FunctionSamples &CallerProfile = ProfileMap[CallerContext];
383 
384     // Since we don't have call count for inlined functions, we
385     // estimate it from inlinee's profile using entry body sample.
386     uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples();
387     // If we don't have samples with location, use 1 to indicate live.
388     if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size())
389       EstimatedCallCount = 1;
390     CallerProfile.addCalledTargetSamples(
391         CallerLeafFrameLoc.Callsite.LineOffset,
392         CallerLeafFrameLoc.Callsite.Discriminator,
393         CalleeProfile.getContext().getName(), EstimatedCallCount);
394     CallerProfile.addBodySamples(CallerLeafFrameLoc.Callsite.LineOffset,
395                                  CallerLeafFrameLoc.Callsite.Discriminator,
396                                  EstimatedCallCount);
397     CallerProfile.addTotalSamples(EstimatedCallCount);
398   }
399 }
400 
401 void CSProfileGenerator::postProcessProfiles() {
402   // Compute hot/cold threshold based on profile. This will be used for cold
403   // context profile merging/trimming.
404   computeSummaryAndThreshold();
405 
406   // Run global pre-inliner to adjust/merge context profile based on estimated
407   // inline decisions.
408   if (EnableCSPreInliner) {
409     CSPreInliner(ProfileMap, *Binary, HotCountThreshold, ColdCountThreshold)
410         .run();
411   }
412 
413   // Trim and merge cold context profile using cold threshold above. By default,
414   // we skip such merging and trimming when preinliner is on.
415   if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() ||
416       CSProfMergeColdContext.getNumOccurrences()) {
417     SampleContextTrimmer(ProfileMap)
418         .trimAndMergeColdContextProfiles(
419             ColdCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext,
420             CSProfMaxColdContextDepth);
421   }
422 }
423 
424 void CSProfileGenerator::computeSummaryAndThreshold() {
425   // Update the default value of cold cutoff for llvm-profgen.
426   // Do it here because we don't want to change the global default,
427   // which would lead CS profile size too large.
428   if (!ProfileSummaryCutoffCold.getNumOccurrences())
429     ProfileSummaryCutoffCold = 999000;
430 
431   SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
432   auto Summary = Builder.computeSummaryForProfiles(ProfileMap);
433   HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold(
434       (Summary->getDetailedSummary()));
435   ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(
436       (Summary->getDetailedSummary()));
437 }
438 
439 void CSProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer,
440                                SampleProfileMap &ProfileMap) {
441   if (std::error_code EC = Writer->write(ProfileMap))
442     exitWithError(std::move(EC));
443 }
444 
445 // Helper function to extract context prefix string stack
446 // Extract context stack for reusing, leaf context stack will
447 // be added compressed while looking up function profile
448 static void extractPrefixContextStack(
449     SampleContextFrameVector &ContextStack,
450     const SmallVectorImpl<const MCDecodedPseudoProbe *> &Probes,
451     ProfiledBinary *Binary) {
452   for (const auto *P : Probes) {
453     Binary->getInlineContextForProbe(P, ContextStack, true);
454   }
455 }
456 
457 void PseudoProbeCSProfileGenerator::generateProfile() {
458   // Enable pseudo probe functionalities in SampleProf
459   FunctionSamples::ProfileIsProbeBased = true;
460   FunctionSamples::ProfileIsCS = true;
461   for (const auto &CI : SampleCounters) {
462     const ProbeBasedCtxKey *CtxKey =
463         dyn_cast<ProbeBasedCtxKey>(CI.first.getPtr());
464     SampleContextFrameVector ContextStack;
465     extractPrefixContextStack(ContextStack, CtxKey->Probes, Binary);
466     // Fill in function body samples from probes, also infer caller's samples
467     // from callee's probe
468     populateBodySamplesWithProbes(CI.second.RangeCounter, ContextStack);
469     // Fill in boundary samples for a call probe
470     populateBoundarySamplesWithProbes(CI.second.BranchCounter, ContextStack);
471   }
472 
473   postProcessProfiles();
474 }
475 
476 void PseudoProbeCSProfileGenerator::extractProbesFromRange(
477     const RangeSample &RangeCounter, ProbeCounterMap &ProbeCounter) {
478   RangeSample Ranges;
479   findDisjointRanges(Ranges, RangeCounter);
480   for (const auto &Range : Ranges) {
481     uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first);
482     uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second);
483     uint64_t Count = Range.second;
484     // Disjoint ranges have introduce zero-filled gap that
485     // doesn't belong to current context, filter them out.
486     if (Count == 0)
487       continue;
488 
489     InstructionPointer IP(Binary, RangeBegin, true);
490 
491     // Disjoint ranges may have range in the middle of two instr,
492     // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range
493     // can be Addr1+1 to Addr2-1. We should ignore such range.
494     if (IP.Address > RangeEnd)
495       continue;
496 
497     while (IP.Address <= RangeEnd) {
498       const AddressProbesMap &Address2ProbesMap =
499           Binary->getAddress2ProbesMap();
500       auto It = Address2ProbesMap.find(IP.Address);
501       if (It != Address2ProbesMap.end()) {
502         for (const auto &Probe : It->second) {
503           if (!Probe.isBlock())
504             continue;
505           ProbeCounter[&Probe] += Count;
506         }
507       }
508 
509       IP.advance();
510     }
511   }
512 }
513 
514 void PseudoProbeCSProfileGenerator::populateBodySamplesWithProbes(
515     const RangeSample &RangeCounter, SampleContextFrames ContextStack) {
516   ProbeCounterMap ProbeCounter;
517   // Extract the top frame probes by looking up each address among the range in
518   // the Address2ProbeMap
519   extractProbesFromRange(RangeCounter, ProbeCounter);
520   std::unordered_map<MCDecodedPseudoProbeInlineTree *,
521                      std::unordered_set<FunctionSamples *>>
522       FrameSamples;
523   for (auto PI : ProbeCounter) {
524     const MCDecodedPseudoProbe *Probe = PI.first;
525     uint64_t Count = PI.second;
526     FunctionSamples &FunctionProfile =
527         getFunctionProfileForLeafProbe(ContextStack, Probe);
528     // Record the current frame and FunctionProfile whenever samples are
529     // collected for non-danglie probes. This is for reporting all of the
530     // zero count probes of the frame later.
531     FrameSamples[Probe->getInlineTreeNode()].insert(&FunctionProfile);
532     FunctionProfile.addBodySamplesForProbe(Probe->getIndex(), Count);
533     FunctionProfile.addTotalSamples(Count);
534     if (Probe->isEntry()) {
535       FunctionProfile.addHeadSamples(Count);
536       // Look up for the caller's function profile
537       const auto *InlinerDesc = Binary->getInlinerDescForProbe(Probe);
538       if (InlinerDesc != nullptr) {
539         // Since the context id will be compressed, we have to use callee's
540         // context id to infer caller's context id to ensure they share the
541         // same context prefix.
542         SampleContextFrames CalleeContextId =
543             FunctionProfile.getContext().getContextFrames();
544         SampleContextFrameVector CallerContextId;
545         SampleContextFrame &&CallerLeafFrameLoc =
546             getCallerContext(CalleeContextId, CallerContextId);
547         uint64_t CallerIndex = CallerLeafFrameLoc.Callsite.LineOffset;
548         assert(CallerIndex &&
549                "Inferred caller's location index shouldn't be zero!");
550         // Save the new context for future references.
551         SampleContextFrames CallerContext =
552             *Contexts.insert(CallerContextId).first;
553         FunctionSamples &CallerProfile =
554             getFunctionProfileForContext(CallerContext);
555         CallerProfile.setFunctionHash(InlinerDesc->FuncHash);
556         CallerProfile.addBodySamples(CallerIndex, 0, Count);
557         CallerProfile.addTotalSamples(Count);
558         CallerProfile.addCalledTargetSamples(
559             CallerIndex, 0, FunctionProfile.getContext().getName(), Count);
560       }
561     }
562   }
563 
564   // Assign zero count for remaining probes without sample hits to
565   // differentiate from probes optimized away, of which the counts are unknown
566   // and will be inferred by the compiler.
567   for (auto &I : FrameSamples) {
568     for (auto *FunctionProfile : I.second) {
569       for (auto *Probe : I.first->getProbes()) {
570         FunctionProfile->addBodySamplesForProbe(Probe->getIndex(), 0);
571       }
572     }
573   }
574 }
575 
576 void PseudoProbeCSProfileGenerator::populateBoundarySamplesWithProbes(
577     const BranchSample &BranchCounter, SampleContextFrames ContextStack) {
578   for (auto BI : BranchCounter) {
579     uint64_t SourceOffset = BI.first.first;
580     uint64_t TargetOffset = BI.first.second;
581     uint64_t Count = BI.second;
582     uint64_t SourceAddress = Binary->offsetToVirtualAddr(SourceOffset);
583     const MCDecodedPseudoProbe *CallProbe =
584         Binary->getCallProbeForAddr(SourceAddress);
585     if (CallProbe == nullptr)
586       continue;
587     FunctionSamples &FunctionProfile =
588         getFunctionProfileForLeafProbe(ContextStack, CallProbe);
589     FunctionProfile.addBodySamples(CallProbe->getIndex(), 0, Count);
590     FunctionProfile.addTotalSamples(Count);
591     StringRef CalleeName = FunctionSamples::getCanonicalFnName(
592         Binary->getFuncFromStartOffset(TargetOffset));
593     if (CalleeName.size() == 0)
594       continue;
595     FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(), 0, CalleeName,
596                                            Count);
597   }
598 }
599 
600 FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe(
601     SampleContextFrames ContextStack, const MCDecodedPseudoProbe *LeafProbe) {
602 
603   // Explicitly copy the context for appending the leaf context
604   SampleContextFrameVector NewContextStack(ContextStack.begin(),
605                                            ContextStack.end());
606   Binary->getInlineContextForProbe(LeafProbe, NewContextStack, true);
607   // For leaf inlined context with the top frame, we should strip off the top
608   // frame's probe id, like:
609   // Inlined stack: [foo:1, bar:2], the ContextId will be "foo:1 @ bar"
610   auto LeafFrame = NewContextStack.back();
611   LeafFrame.Callsite = LineLocation(0, 0);
612   NewContextStack.pop_back();
613   // Compress the context string except for the leaf frame
614   CSProfileGenerator::compressRecursionContext(NewContextStack);
615   CSProfileGenerator::trimContext(NewContextStack);
616   NewContextStack.push_back(LeafFrame);
617   // Save the new context for future references.
618   SampleContextFrames NewContext = *Contexts.insert(NewContextStack).first;
619 
620   const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->getGuid());
621   bool WasLeafInlined = LeafProbe->getInlineTreeNode()->hasInlineSite();
622   FunctionSamples &FunctionProile =
623       getFunctionProfileForContext(NewContext, WasLeafInlined);
624   FunctionProile.setFunctionHash(FuncDesc->FuncHash);
625   return FunctionProile;
626 }
627 
628 } // end namespace sampleprof
629 } // end namespace llvm
630