1 //===-- ProfileGenerator.cpp - Profile Generator ---------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ProfileGenerator.h" 10 #include "ProfiledBinary.h" 11 #include "llvm/ProfileData/ProfileCommon.h" 12 #include <unordered_set> 13 14 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 15 cl::Required, 16 cl::desc("Output profile file")); 17 static cl::alias OutputA("o", cl::desc("Alias for --output"), 18 cl::aliasopt(OutputFilename)); 19 20 static cl::opt<SampleProfileFormat> OutputFormat( 21 "format", cl::desc("Format of output profile"), cl::init(SPF_Ext_Binary), 22 cl::values( 23 clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"), 24 clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"), 25 clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"), 26 clEnumValN(SPF_Text, "text", "Text encoding"), 27 clEnumValN(SPF_GCC, "gcc", 28 "GCC encoding (only meaningful for -sample)"))); 29 30 cl::opt<bool> UseMD5( 31 "use-md5", cl::init(false), cl::Hidden, 32 cl::desc("Use md5 to represent function names in the output profile (only " 33 "meaningful for -extbinary)")); 34 35 static cl::opt<int32_t, true> RecursionCompression( 36 "compress-recursion", 37 cl::desc("Compressing recursion by deduplicating adjacent frame " 38 "sequences up to the specified size. -1 means no size limit."), 39 cl::Hidden, 40 cl::location(llvm::sampleprof::CSProfileGenerator::MaxCompressionSize)); 41 42 static cl::opt<bool> CSProfMergeColdContext( 43 "csprof-merge-cold-context", cl::init(true), cl::ZeroOrMore, 44 cl::desc("If the total count of context profile is smaller than " 45 "the threshold, it will be merged into context-less base " 46 "profile.")); 47 48 static cl::opt<bool> CSProfTrimColdContext( 49 "csprof-trim-cold-context", cl::init(false), cl::ZeroOrMore, 50 cl::desc("If the total count of the profile after all merge is done " 51 "is still smaller than threshold, it will be trimmed.")); 52 53 static cl::opt<uint32_t> CSProfMaxColdContextDepth( 54 "csprof-max-cold-context-depth", cl::init(1), cl::ZeroOrMore, 55 cl::desc("Keep the last K contexts while merging cold profile. 1 means the " 56 "context-less base profile")); 57 58 static cl::opt<int, true> CSProfMaxContextDepth( 59 "csprof-max-context-depth", cl::ZeroOrMore, 60 cl::desc("Keep the last K contexts while merging profile. -1 means no " 61 "depth limit."), 62 cl::location(llvm::sampleprof::CSProfileGenerator::MaxContextDepth)); 63 64 extern cl::opt<int> ProfileSummaryCutoffCold; 65 66 using namespace llvm; 67 using namespace sampleprof; 68 69 namespace llvm { 70 namespace sampleprof { 71 72 // Initialize the MaxCompressionSize to -1 which means no size limit 73 int32_t CSProfileGenerator::MaxCompressionSize = -1; 74 75 int CSProfileGenerator::MaxContextDepth = -1; 76 77 std::unique_ptr<ProfileGenerator> 78 ProfileGenerator::create(ProfiledBinary *Binary, 79 const ContextSampleCounterMap &SampleCounters, 80 enum PerfScriptType SampleType) { 81 std::unique_ptr<ProfileGenerator> ProfileGenerator; 82 if (SampleType == PERF_LBR_STACK) { 83 if (Binary->usePseudoProbes()) { 84 ProfileGenerator.reset( 85 new PseudoProbeCSProfileGenerator(Binary, SampleCounters)); 86 } else { 87 ProfileGenerator.reset(new CSProfileGenerator(Binary, SampleCounters)); 88 } 89 } else { 90 // TODO: 91 llvm_unreachable("Unsupported perfscript!"); 92 } 93 94 return ProfileGenerator; 95 } 96 97 void ProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer, 98 SampleProfileMap &ProfileMap) { 99 if (std::error_code EC = Writer->write(ProfileMap)) 100 exitWithError(std::move(EC)); 101 } 102 103 void ProfileGenerator::write() { 104 auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat); 105 if (std::error_code EC = WriterOrErr.getError()) 106 exitWithError(EC, OutputFilename); 107 108 if (UseMD5) { 109 if (OutputFormat != SPF_Ext_Binary) 110 WithColor::warning() << "-use-md5 is ignored. Specify " 111 "--format=extbinary to enable it\n"; 112 else 113 WriterOrErr.get()->setUseMD5(); 114 } 115 116 write(std::move(WriterOrErr.get()), ProfileMap); 117 } 118 119 void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges, 120 const RangeSample &Ranges) { 121 122 /* 123 Regions may overlap with each other. Using the boundary info, find all 124 disjoint ranges and their sample count. BoundaryPoint contains the count 125 multiple samples begin/end at this points. 126 127 |<--100-->| Sample1 128 |<------200------>| Sample2 129 A B C 130 131 In the example above, 132 Sample1 begins at A, ends at B, its value is 100. 133 Sample2 beings at A, ends at C, its value is 200. 134 For A, BeginCount is the sum of sample begins at A, which is 300 and no 135 samples ends at A, so EndCount is 0. 136 Then boundary points A, B, and C with begin/end counts are: 137 A: (300, 0) 138 B: (0, 100) 139 C: (0, 200) 140 */ 141 struct BoundaryPoint { 142 // Sum of sample counts beginning at this point 143 uint64_t BeginCount; 144 // Sum of sample counts ending at this point 145 uint64_t EndCount; 146 147 BoundaryPoint() : BeginCount(0), EndCount(0){}; 148 149 void addBeginCount(uint64_t Count) { BeginCount += Count; } 150 151 void addEndCount(uint64_t Count) { EndCount += Count; } 152 }; 153 154 /* 155 For the above example. With boundary points, follwing logic finds two 156 disjoint region of 157 158 [A,B]: 300 159 [B+1,C]: 200 160 161 If there is a boundary point that both begin and end, the point itself 162 becomes a separate disjoint region. For example, if we have original 163 ranges of 164 165 |<--- 100 --->| 166 |<--- 200 --->| 167 A B C 168 169 there are three boundary points with their begin/end counts of 170 171 A: (100, 0) 172 B: (200, 100) 173 C: (0, 200) 174 175 the disjoint ranges would be 176 177 [A, B-1]: 100 178 [B, B]: 300 179 [B+1, C]: 200. 180 */ 181 std::map<uint64_t, BoundaryPoint> Boundaries; 182 183 for (auto Item : Ranges) { 184 uint64_t Begin = Item.first.first; 185 uint64_t End = Item.first.second; 186 assert(Begin <= End && "Invalid instruction range"); 187 uint64_t Count = Item.second; 188 if (Boundaries.find(Begin) == Boundaries.end()) 189 Boundaries[Begin] = BoundaryPoint(); 190 Boundaries[Begin].addBeginCount(Count); 191 192 if (Boundaries.find(End) == Boundaries.end()) 193 Boundaries[End] = BoundaryPoint(); 194 Boundaries[End].addEndCount(Count); 195 } 196 197 uint64_t BeginAddress = UINT64_MAX; 198 uint64_t Count = 0; 199 for (auto Item : Boundaries) { 200 uint64_t Address = Item.first; 201 BoundaryPoint &Point = Item.second; 202 if (Point.BeginCount) { 203 if (BeginAddress != UINT64_MAX) 204 DisjointRanges[{BeginAddress, Address - 1}] = Count; 205 Count += Point.BeginCount; 206 BeginAddress = Address; 207 } 208 if (Point.EndCount) { 209 assert((BeginAddress != UINT64_MAX) && 210 "First boundary point cannot be 'end' point"); 211 DisjointRanges[{BeginAddress, Address}] = Count; 212 assert(Count >= Point.EndCount && "Mismatched live ranges"); 213 Count -= Point.EndCount; 214 BeginAddress = Address + 1; 215 } 216 } 217 } 218 219 FunctionSamples &CSProfileGenerator::getFunctionProfileForContext( 220 const SampleContextFrameVector &Context, bool WasLeafInlined) { 221 auto I = ProfileMap.find(SampleContext(Context)); 222 if (I == ProfileMap.end()) { 223 // Save the new context for future references. 224 SampleContextFrames NewContext = *Contexts.insert(Context).first; 225 SampleContext FContext(NewContext, RawContext); 226 auto Ret = ProfileMap.emplace(FContext, FunctionSamples()); 227 if (WasLeafInlined) 228 FContext.setAttribute(ContextWasInlined); 229 FunctionSamples &FProfile = Ret.first->second; 230 FProfile.setContext(FContext); 231 return Ret.first->second; 232 } 233 return I->second; 234 } 235 236 void CSProfileGenerator::generateProfile() { 237 FunctionSamples::ProfileIsCS = true; 238 for (const auto &CI : SampleCounters) { 239 const StringBasedCtxKey *CtxKey = 240 dyn_cast<StringBasedCtxKey>(CI.first.getPtr()); 241 // Get or create function profile for the range 242 FunctionSamples &FunctionProfile = 243 getFunctionProfileForContext(CtxKey->Context, CtxKey->WasLeafInlined); 244 245 // Fill in function body samples 246 populateFunctionBodySamples(FunctionProfile, CI.second.RangeCounter); 247 // Fill in boundary sample counts as well as call site samples for calls 248 populateFunctionBoundarySamples(CtxKey->Context, FunctionProfile, 249 CI.second.BranchCounter); 250 } 251 // Fill in call site value sample for inlined calls and also use context to 252 // infer missing samples. Since we don't have call count for inlined 253 // functions, we estimate it from inlinee's profile using the entry of the 254 // body sample. 255 populateInferredFunctionSamples(); 256 257 postProcessProfiles(); 258 } 259 260 void CSProfileGenerator::updateBodySamplesforFunctionProfile( 261 FunctionSamples &FunctionProfile, const SampleContextFrame &LeafLoc, 262 uint64_t Count) { 263 // Filter out invalid negative(int type) lineOffset 264 if (LeafLoc.Callsite.LineOffset & 0x80000000) 265 return; 266 // Use the maximum count of samples with same line location 267 ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt( 268 LeafLoc.Callsite.LineOffset, LeafLoc.Callsite.Discriminator); 269 uint64_t PreviousCount = R ? R.get() : 0; 270 if (PreviousCount < Count) { 271 FunctionProfile.addBodySamples(LeafLoc.Callsite.LineOffset, 272 LeafLoc.Callsite.Discriminator, 273 Count - PreviousCount); 274 } 275 } 276 277 void CSProfileGenerator::populateFunctionBodySamples( 278 FunctionSamples &FunctionProfile, const RangeSample &RangeCounter) { 279 // Compute disjoint ranges first, so we can use MAX 280 // for calculating count for each location. 281 RangeSample Ranges; 282 findDisjointRanges(Ranges, RangeCounter); 283 for (auto Range : Ranges) { 284 uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); 285 uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); 286 uint64_t Count = Range.second; 287 // Disjoint ranges have introduce zero-filled gap that 288 // doesn't belong to current context, filter them out. 289 if (Count == 0) 290 continue; 291 292 InstructionPointer IP(Binary, RangeBegin, true); 293 294 // Disjoint ranges may have range in the middle of two instr, 295 // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range 296 // can be Addr1+1 to Addr2-1. We should ignore such range. 297 if (IP.Address > RangeEnd) 298 continue; 299 300 while (IP.Address <= RangeEnd) { 301 uint64_t Offset = Binary->virtualAddrToOffset(IP.Address); 302 auto LeafLoc = Binary->getInlineLeafFrameLoc(Offset); 303 if (LeafLoc.hasValue()) { 304 // Recording body sample for this specific context 305 updateBodySamplesforFunctionProfile(FunctionProfile, *LeafLoc, Count); 306 } 307 // Accumulate total sample count even it's a line with invalid debug info 308 FunctionProfile.addTotalSamples(Count); 309 // Move to next IP within the range 310 IP.advance(); 311 } 312 } 313 } 314 315 void CSProfileGenerator::populateFunctionBoundarySamples( 316 SampleContextFrames ContextId, FunctionSamples &FunctionProfile, 317 const BranchSample &BranchCounters) { 318 319 for (auto Entry : BranchCounters) { 320 uint64_t SourceOffset = Entry.first.first; 321 uint64_t TargetOffset = Entry.first.second; 322 uint64_t Count = Entry.second; 323 // Get the callee name by branch target if it's a call branch 324 StringRef CalleeName = FunctionSamples::getCanonicalFnName( 325 Binary->getFuncFromStartOffset(TargetOffset)); 326 if (CalleeName.size() == 0) 327 continue; 328 329 // Record called target sample and its count 330 auto LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset); 331 if (!LeafLoc.hasValue()) 332 continue; 333 FunctionProfile.addCalledTargetSamples(LeafLoc->Callsite.LineOffset, 334 LeafLoc->Callsite.Discriminator, 335 CalleeName, Count); 336 337 // Record head sample for called target(callee) 338 SampleContextFrameVector CalleeCtx(ContextId.begin(), ContextId.end()); 339 assert(CalleeCtx.back().CallerName == LeafLoc->CallerName && 340 "Leaf function name doesn't match"); 341 CalleeCtx.back() = *LeafLoc; 342 CalleeCtx.emplace_back(CalleeName, LineLocation(0, 0)); 343 FunctionSamples &CalleeProfile = getFunctionProfileForContext(CalleeCtx); 344 assert(Count != 0 && "Unexpected zero weight branch"); 345 CalleeProfile.addHeadSamples(Count); 346 } 347 } 348 349 static SampleContextFrame 350 getCallerContext(SampleContextFrames CalleeContext, 351 SampleContextFrameVector &CallerContext) { 352 assert(CalleeContext.size() > 1 && "Unexpected empty context"); 353 CalleeContext = CalleeContext.drop_back(); 354 CallerContext.assign(CalleeContext.begin(), CalleeContext.end()); 355 SampleContextFrame CallerFrame = CallerContext.back(); 356 CallerContext.back().Callsite = LineLocation(0, 0); 357 return CallerFrame; 358 } 359 360 void CSProfileGenerator::populateInferredFunctionSamples() { 361 for (const auto &Item : ProfileMap) { 362 const auto &CalleeContext = Item.first; 363 const FunctionSamples &CalleeProfile = Item.second; 364 365 // If we already have head sample counts, we must have value profile 366 // for call sites added already. Skip to avoid double counting. 367 if (CalleeProfile.getHeadSamples()) 368 continue; 369 // If we don't have context, nothing to do for caller's call site. 370 // This could happen for entry point function. 371 if (CalleeContext.isBaseContext()) 372 continue; 373 374 // Infer Caller's frame loc and context ID through string splitting 375 SampleContextFrameVector CallerContextId; 376 SampleContextFrame &&CallerLeafFrameLoc = 377 getCallerContext(CalleeContext.getContextFrames(), CallerContextId); 378 SampleContextFrames CallerContext(CallerContextId); 379 380 // It's possible that we haven't seen any sample directly in the caller, 381 // in which case CallerProfile will not exist. But we can't modify 382 // ProfileMap while iterating it. 383 // TODO: created function profile for those callers too 384 if (ProfileMap.find(CallerContext) == ProfileMap.end()) 385 continue; 386 FunctionSamples &CallerProfile = ProfileMap[CallerContext]; 387 388 // Since we don't have call count for inlined functions, we 389 // estimate it from inlinee's profile using entry body sample. 390 uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples(); 391 // If we don't have samples with location, use 1 to indicate live. 392 if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size()) 393 EstimatedCallCount = 1; 394 CallerProfile.addCalledTargetSamples( 395 CallerLeafFrameLoc.Callsite.LineOffset, 396 CallerLeafFrameLoc.Callsite.Discriminator, 397 CalleeProfile.getContext().getName(), EstimatedCallCount); 398 CallerProfile.addBodySamples(CallerLeafFrameLoc.Callsite.LineOffset, 399 CallerLeafFrameLoc.Callsite.Discriminator, 400 EstimatedCallCount); 401 CallerProfile.addTotalSamples(EstimatedCallCount); 402 } 403 } 404 405 void CSProfileGenerator::postProcessProfiles() { 406 // Compute hot/cold threshold based on profile. This will be used for cold 407 // context profile merging/trimming. 408 computeSummaryAndThreshold(); 409 410 // Run global pre-inliner to adjust/merge context profile based on estimated 411 // inline decisions. 412 if (EnableCSPreInliner) { 413 CSPreInliner(ProfileMap, *Binary, HotCountThreshold, ColdCountThreshold) 414 .run(); 415 } 416 417 // Trim and merge cold context profile using cold threshold above. By default, 418 // we skip such merging and trimming when preinliner is on. 419 if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() || 420 CSProfMergeColdContext.getNumOccurrences()) { 421 SampleContextTrimmer(ProfileMap) 422 .trimAndMergeColdContextProfiles( 423 HotCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, 424 CSProfMaxColdContextDepth); 425 } 426 } 427 428 void CSProfileGenerator::computeSummaryAndThreshold() { 429 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); 430 auto Summary = Builder.computeSummaryForProfiles(ProfileMap); 431 HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold( 432 (Summary->getDetailedSummary())); 433 ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold( 434 (Summary->getDetailedSummary())); 435 } 436 437 void CSProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer, 438 SampleProfileMap &ProfileMap) { 439 if (std::error_code EC = Writer->write(ProfileMap)) 440 exitWithError(std::move(EC)); 441 } 442 443 // Helper function to extract context prefix string stack 444 // Extract context stack for reusing, leaf context stack will 445 // be added compressed while looking up function profile 446 static void extractPrefixContextStack( 447 SampleContextFrameVector &ContextStack, 448 const SmallVectorImpl<const MCDecodedPseudoProbe *> &Probes, 449 ProfiledBinary *Binary) { 450 for (const auto *P : Probes) { 451 Binary->getInlineContextForProbe(P, ContextStack, true); 452 } 453 } 454 455 void PseudoProbeCSProfileGenerator::generateProfile() { 456 // Enable pseudo probe functionalities in SampleProf 457 FunctionSamples::ProfileIsProbeBased = true; 458 FunctionSamples::ProfileIsCS = true; 459 for (const auto &CI : SampleCounters) { 460 const ProbeBasedCtxKey *CtxKey = 461 dyn_cast<ProbeBasedCtxKey>(CI.first.getPtr()); 462 SampleContextFrameVector ContextStack; 463 extractPrefixContextStack(ContextStack, CtxKey->Probes, Binary); 464 // Fill in function body samples from probes, also infer caller's samples 465 // from callee's probe 466 populateBodySamplesWithProbes(CI.second.RangeCounter, ContextStack); 467 // Fill in boundary samples for a call probe 468 populateBoundarySamplesWithProbes(CI.second.BranchCounter, ContextStack); 469 } 470 471 postProcessProfiles(); 472 } 473 474 void PseudoProbeCSProfileGenerator::extractProbesFromRange( 475 const RangeSample &RangeCounter, ProbeCounterMap &ProbeCounter) { 476 RangeSample Ranges; 477 findDisjointRanges(Ranges, RangeCounter); 478 for (const auto &Range : Ranges) { 479 uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); 480 uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); 481 uint64_t Count = Range.second; 482 // Disjoint ranges have introduce zero-filled gap that 483 // doesn't belong to current context, filter them out. 484 if (Count == 0) 485 continue; 486 487 InstructionPointer IP(Binary, RangeBegin, true); 488 489 // Disjoint ranges may have range in the middle of two instr, 490 // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range 491 // can be Addr1+1 to Addr2-1. We should ignore such range. 492 if (IP.Address > RangeEnd) 493 continue; 494 495 while (IP.Address <= RangeEnd) { 496 const AddressProbesMap &Address2ProbesMap = 497 Binary->getAddress2ProbesMap(); 498 auto It = Address2ProbesMap.find(IP.Address); 499 if (It != Address2ProbesMap.end()) { 500 for (const auto &Probe : It->second) { 501 if (!Probe.isBlock()) 502 continue; 503 ProbeCounter[&Probe] += Count; 504 } 505 } 506 507 IP.advance(); 508 } 509 } 510 } 511 512 void PseudoProbeCSProfileGenerator::populateBodySamplesWithProbes( 513 const RangeSample &RangeCounter, SampleContextFrames ContextStack) { 514 ProbeCounterMap ProbeCounter; 515 // Extract the top frame probes by looking up each address among the range in 516 // the Address2ProbeMap 517 extractProbesFromRange(RangeCounter, ProbeCounter); 518 std::unordered_map<MCDecodedPseudoProbeInlineTree *, 519 std::unordered_set<FunctionSamples *>> 520 FrameSamples; 521 for (auto PI : ProbeCounter) { 522 const MCDecodedPseudoProbe *Probe = PI.first; 523 uint64_t Count = PI.second; 524 FunctionSamples &FunctionProfile = 525 getFunctionProfileForLeafProbe(ContextStack, Probe); 526 // Record the current frame and FunctionProfile whenever samples are 527 // collected for non-danglie probes. This is for reporting all of the 528 // zero count probes of the frame later. 529 FrameSamples[Probe->getInlineTreeNode()].insert(&FunctionProfile); 530 FunctionProfile.addBodySamplesForProbe(Probe->getIndex(), Count); 531 FunctionProfile.addTotalSamples(Count); 532 if (Probe->isEntry()) { 533 FunctionProfile.addHeadSamples(Count); 534 // Look up for the caller's function profile 535 const auto *InlinerDesc = Binary->getInlinerDescForProbe(Probe); 536 if (InlinerDesc != nullptr) { 537 // Since the context id will be compressed, we have to use callee's 538 // context id to infer caller's context id to ensure they share the 539 // same context prefix. 540 SampleContextFrames CalleeContextId = 541 FunctionProfile.getContext().getContextFrames(); 542 SampleContextFrameVector CallerContextId; 543 SampleContextFrame &&CallerLeafFrameLoc = 544 getCallerContext(CalleeContextId, CallerContextId); 545 uint64_t CallerIndex = CallerLeafFrameLoc.Callsite.LineOffset; 546 assert(CallerIndex && 547 "Inferred caller's location index shouldn't be zero!"); 548 FunctionSamples &CallerProfile = 549 getFunctionProfileForContext(CallerContextId); 550 CallerProfile.setFunctionHash(InlinerDesc->FuncHash); 551 CallerProfile.addBodySamples(CallerIndex, 0, Count); 552 CallerProfile.addTotalSamples(Count); 553 CallerProfile.addCalledTargetSamples( 554 CallerIndex, 0, FunctionProfile.getContext().getName(), Count); 555 } 556 } 557 } 558 559 // Assign zero count for remaining probes without sample hits to 560 // differentiate from probes optimized away, of which the counts are unknown 561 // and will be inferred by the compiler. 562 for (auto &I : FrameSamples) { 563 for (auto *FunctionProfile : I.second) { 564 for (auto *Probe : I.first->getProbes()) { 565 FunctionProfile->addBodySamplesForProbe(Probe->getIndex(), 0); 566 } 567 } 568 } 569 } 570 571 void PseudoProbeCSProfileGenerator::populateBoundarySamplesWithProbes( 572 const BranchSample &BranchCounter, SampleContextFrames ContextStack) { 573 for (auto BI : BranchCounter) { 574 uint64_t SourceOffset = BI.first.first; 575 uint64_t TargetOffset = BI.first.second; 576 uint64_t Count = BI.second; 577 uint64_t SourceAddress = Binary->offsetToVirtualAddr(SourceOffset); 578 const MCDecodedPseudoProbe *CallProbe = 579 Binary->getCallProbeForAddr(SourceAddress); 580 if (CallProbe == nullptr) 581 continue; 582 FunctionSamples &FunctionProfile = 583 getFunctionProfileForLeafProbe(ContextStack, CallProbe); 584 FunctionProfile.addBodySamples(CallProbe->getIndex(), 0, Count); 585 FunctionProfile.addTotalSamples(Count); 586 StringRef CalleeName = FunctionSamples::getCanonicalFnName( 587 Binary->getFuncFromStartOffset(TargetOffset)); 588 if (CalleeName.size() == 0) 589 continue; 590 FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(), 0, CalleeName, 591 Count); 592 } 593 } 594 595 FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe( 596 SampleContextFrames ContextStack, const MCDecodedPseudoProbe *LeafProbe) { 597 598 // Explicitly copy the context for appending the leaf context 599 SampleContextFrameVector NewContextStack(ContextStack.begin(), 600 ContextStack.end()); 601 Binary->getInlineContextForProbe(LeafProbe, NewContextStack, true); 602 // For leaf inlined context with the top frame, we should strip off the top 603 // frame's probe id, like: 604 // Inlined stack: [foo:1, bar:2], the ContextId will be "foo:1 @ bar" 605 auto LeafFrame = NewContextStack.back(); 606 LeafFrame.Callsite = LineLocation(0, 0); 607 NewContextStack.pop_back(); 608 // Compress the context string except for the leaf frame 609 CSProfileGenerator::compressRecursionContext(NewContextStack); 610 CSProfileGenerator::trimContext(NewContextStack); 611 NewContextStack.push_back(LeafFrame); 612 613 const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->getGuid()); 614 bool WasLeafInlined = LeafProbe->getInlineTreeNode()->hasInlineSite(); 615 FunctionSamples &FunctionProile = 616 getFunctionProfileForContext(NewContextStack, WasLeafInlined); 617 FunctionProile.setFunctionHash(FuncDesc->FuncHash); 618 return FunctionProile; 619 } 620 621 } // end namespace sampleprof 622 } // end namespace llvm 623