1 //===-- ProfileGenerator.cpp - Profile Generator ---------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ProfileGenerator.h" 10 #include "ProfiledBinary.h" 11 #include "llvm/ProfileData/ProfileCommon.h" 12 #include <unordered_set> 13 14 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 15 cl::Required, 16 cl::desc("Output profile file")); 17 static cl::alias OutputA("o", cl::desc("Alias for --output"), 18 cl::aliasopt(OutputFilename)); 19 20 static cl::opt<SampleProfileFormat> OutputFormat( 21 "format", cl::desc("Format of output profile"), cl::init(SPF_Ext_Binary), 22 cl::values( 23 clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"), 24 clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"), 25 clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"), 26 clEnumValN(SPF_Text, "text", "Text encoding"), 27 clEnumValN(SPF_GCC, "gcc", 28 "GCC encoding (only meaningful for -sample)"))); 29 30 cl::opt<bool> UseMD5( 31 "use-md5", cl::init(false), cl::Hidden, 32 cl::desc("Use md5 to represent function names in the output profile (only " 33 "meaningful for -extbinary)")); 34 35 static cl::opt<int32_t, true> RecursionCompression( 36 "compress-recursion", 37 cl::desc("Compressing recursion by deduplicating adjacent frame " 38 "sequences up to the specified size. -1 means no size limit."), 39 cl::Hidden, 40 cl::location(llvm::sampleprof::CSProfileGenerator::MaxCompressionSize)); 41 42 static cl::opt<bool> CSProfMergeColdContext( 43 "csprof-merge-cold-context", cl::init(true), cl::ZeroOrMore, 44 cl::desc("If the total count of context profile is smaller than " 45 "the threshold, it will be merged into context-less base " 46 "profile.")); 47 48 static cl::opt<bool> CSProfTrimColdContext( 49 "csprof-trim-cold-context", cl::init(false), cl::ZeroOrMore, 50 cl::desc("If the total count of the profile after all merge is done " 51 "is still smaller than threshold, it will be trimmed.")); 52 53 static cl::opt<uint32_t> CSProfMaxColdContextDepth( 54 "csprof-max-cold-context-depth", cl::init(1), cl::ZeroOrMore, 55 cl::desc("Keep the last K contexts while merging cold profile. 1 means the " 56 "context-less base profile")); 57 58 static cl::opt<int, true> CSProfMaxContextDepth( 59 "csprof-max-context-depth", cl::ZeroOrMore, 60 cl::desc("Keep the last K contexts while merging profile. -1 means no " 61 "depth limit."), 62 cl::location(llvm::sampleprof::CSProfileGenerator::MaxContextDepth)); 63 64 extern cl::opt<int> ProfileSummaryCutoffCold; 65 66 using namespace llvm; 67 using namespace sampleprof; 68 69 namespace llvm { 70 namespace sampleprof { 71 72 // Initialize the MaxCompressionSize to -1 which means no size limit 73 int32_t CSProfileGenerator::MaxCompressionSize = -1; 74 75 int CSProfileGenerator::MaxContextDepth = -1; 76 77 std::unique_ptr<ProfileGenerator> 78 ProfileGenerator::create(ProfiledBinary *Binary, 79 const ContextSampleCounterMap &SampleCounters, 80 enum PerfScriptType SampleType) { 81 std::unique_ptr<ProfileGenerator> ProfileGenerator; 82 if (SampleType == PERF_LBR_STACK) { 83 if (Binary->usePseudoProbes()) { 84 ProfileGenerator.reset( 85 new PseudoProbeCSProfileGenerator(Binary, SampleCounters)); 86 } else { 87 ProfileGenerator.reset(new CSProfileGenerator(Binary, SampleCounters)); 88 } 89 } else { 90 // TODO: 91 llvm_unreachable("Unsupported perfscript!"); 92 } 93 94 return ProfileGenerator; 95 } 96 97 void ProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer, 98 SampleProfileMap &ProfileMap) { 99 if (std::error_code EC = Writer->write(ProfileMap)) 100 exitWithError(std::move(EC)); 101 } 102 103 void ProfileGenerator::write() { 104 auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat); 105 if (std::error_code EC = WriterOrErr.getError()) 106 exitWithError(EC, OutputFilename); 107 108 if (UseMD5) { 109 if (OutputFormat != SPF_Ext_Binary) 110 WithColor::warning() << "-use-md5 is ignored. Specify " 111 "--format=extbinary to enable it\n"; 112 else 113 WriterOrErr.get()->setUseMD5(); 114 } 115 116 write(std::move(WriterOrErr.get()), ProfileMap); 117 } 118 119 void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges, 120 const RangeSample &Ranges) { 121 122 /* 123 Regions may overlap with each other. Using the boundary info, find all 124 disjoint ranges and their sample count. BoundaryPoint contains the count 125 multiple samples begin/end at this points. 126 127 |<--100-->| Sample1 128 |<------200------>| Sample2 129 A B C 130 131 In the example above, 132 Sample1 begins at A, ends at B, its value is 100. 133 Sample2 beings at A, ends at C, its value is 200. 134 For A, BeginCount is the sum of sample begins at A, which is 300 and no 135 samples ends at A, so EndCount is 0. 136 Then boundary points A, B, and C with begin/end counts are: 137 A: (300, 0) 138 B: (0, 100) 139 C: (0, 200) 140 */ 141 struct BoundaryPoint { 142 // Sum of sample counts beginning at this point 143 uint64_t BeginCount; 144 // Sum of sample counts ending at this point 145 uint64_t EndCount; 146 147 BoundaryPoint() : BeginCount(0), EndCount(0){}; 148 149 void addBeginCount(uint64_t Count) { BeginCount += Count; } 150 151 void addEndCount(uint64_t Count) { EndCount += Count; } 152 }; 153 154 /* 155 For the above example. With boundary points, follwing logic finds two 156 disjoint region of 157 158 [A,B]: 300 159 [B+1,C]: 200 160 161 If there is a boundary point that both begin and end, the point itself 162 becomes a separate disjoint region. For example, if we have original 163 ranges of 164 165 |<--- 100 --->| 166 |<--- 200 --->| 167 A B C 168 169 there are three boundary points with their begin/end counts of 170 171 A: (100, 0) 172 B: (200, 100) 173 C: (0, 200) 174 175 the disjoint ranges would be 176 177 [A, B-1]: 100 178 [B, B]: 300 179 [B+1, C]: 200. 180 */ 181 std::map<uint64_t, BoundaryPoint> Boundaries; 182 183 for (auto Item : Ranges) { 184 uint64_t Begin = Item.first.first; 185 uint64_t End = Item.first.second; 186 uint64_t Count = Item.second; 187 if (Boundaries.find(Begin) == Boundaries.end()) 188 Boundaries[Begin] = BoundaryPoint(); 189 Boundaries[Begin].addBeginCount(Count); 190 191 if (Boundaries.find(End) == Boundaries.end()) 192 Boundaries[End] = BoundaryPoint(); 193 Boundaries[End].addEndCount(Count); 194 } 195 196 uint64_t BeginAddress = UINT64_MAX; 197 int Count = 0; 198 for (auto Item : Boundaries) { 199 uint64_t Address = Item.first; 200 BoundaryPoint &Point = Item.second; 201 if (Point.BeginCount) { 202 if (BeginAddress != UINT64_MAX) 203 DisjointRanges[{BeginAddress, Address - 1}] = Count; 204 Count += Point.BeginCount; 205 BeginAddress = Address; 206 } 207 if (Point.EndCount) { 208 assert((BeginAddress != UINT64_MAX) && 209 "First boundary point cannot be 'end' point"); 210 DisjointRanges[{BeginAddress, Address}] = Count; 211 Count -= Point.EndCount; 212 BeginAddress = Address + 1; 213 } 214 } 215 } 216 217 FunctionSamples & 218 CSProfileGenerator::getFunctionProfileForContext(SampleContextFrames Context, 219 bool WasLeafInlined) { 220 SampleContext FContext(Context); 221 auto Ret = ProfileMap.emplace(Context, FunctionSamples()); 222 if (Ret.second) { 223 SampleContext FContext(Context, RawContext); 224 if (WasLeafInlined) 225 FContext.setAttribute(ContextWasInlined); 226 FunctionSamples &FProfile = Ret.first->second; 227 FProfile.setContext(FContext); 228 } 229 return Ret.first->second; 230 } 231 232 void CSProfileGenerator::generateProfile() { 233 FunctionSamples::ProfileIsCS = true; 234 for (const auto &CI : SampleCounters) { 235 const StringBasedCtxKey *CtxKey = 236 dyn_cast<StringBasedCtxKey>(CI.first.getPtr()); 237 // Get or create function profile for the range 238 FunctionSamples &FunctionProfile = 239 getFunctionProfileForContext(CtxKey->Context, CtxKey->WasLeafInlined); 240 241 // Fill in function body samples 242 populateFunctionBodySamples(FunctionProfile, CI.second.RangeCounter); 243 // Fill in boundary sample counts as well as call site samples for calls 244 populateFunctionBoundarySamples(CtxKey->Context, FunctionProfile, 245 CI.second.BranchCounter); 246 } 247 // Fill in call site value sample for inlined calls and also use context to 248 // infer missing samples. Since we don't have call count for inlined 249 // functions, we estimate it from inlinee's profile using the entry of the 250 // body sample. 251 populateInferredFunctionSamples(); 252 253 postProcessProfiles(); 254 } 255 256 void CSProfileGenerator::updateBodySamplesforFunctionProfile( 257 FunctionSamples &FunctionProfile, const SampleContextFrame &LeafLoc, 258 uint64_t Count) { 259 // Filter out invalid negative(int type) lineOffset 260 if (LeafLoc.Callsite.LineOffset & 0x80000000) 261 return; 262 // Use the maximum count of samples with same line location 263 ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt( 264 LeafLoc.Callsite.LineOffset, LeafLoc.Callsite.Discriminator); 265 uint64_t PreviousCount = R ? R.get() : 0; 266 if (PreviousCount < Count) { 267 FunctionProfile.addBodySamples(LeafLoc.Callsite.LineOffset, 268 LeafLoc.Callsite.Discriminator, 269 Count - PreviousCount); 270 } 271 } 272 273 void CSProfileGenerator::populateFunctionBodySamples( 274 FunctionSamples &FunctionProfile, const RangeSample &RangeCounter) { 275 // Compute disjoint ranges first, so we can use MAX 276 // for calculating count for each location. 277 RangeSample Ranges; 278 findDisjointRanges(Ranges, RangeCounter); 279 for (auto Range : Ranges) { 280 uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); 281 uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); 282 uint64_t Count = Range.second; 283 // Disjoint ranges have introduce zero-filled gap that 284 // doesn't belong to current context, filter them out. 285 if (Count == 0) 286 continue; 287 288 InstructionPointer IP(Binary, RangeBegin, true); 289 290 // Disjoint ranges may have range in the middle of two instr, 291 // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range 292 // can be Addr1+1 to Addr2-1. We should ignore such range. 293 if (IP.Address > RangeEnd) 294 continue; 295 296 while (IP.Address <= RangeEnd) { 297 uint64_t Offset = Binary->virtualAddrToOffset(IP.Address); 298 auto LeafLoc = Binary->getInlineLeafFrameLoc(Offset); 299 if (LeafLoc.hasValue()) { 300 // Recording body sample for this specific context 301 updateBodySamplesforFunctionProfile(FunctionProfile, *LeafLoc, Count); 302 } 303 // Accumulate total sample count even it's a line with invalid debug info 304 FunctionProfile.addTotalSamples(Count); 305 // Move to next IP within the range 306 IP.advance(); 307 } 308 } 309 } 310 311 void CSProfileGenerator::populateFunctionBoundarySamples( 312 SampleContextFrames ContextId, FunctionSamples &FunctionProfile, 313 const BranchSample &BranchCounters) { 314 315 for (auto Entry : BranchCounters) { 316 uint64_t SourceOffset = Entry.first.first; 317 uint64_t TargetOffset = Entry.first.second; 318 uint64_t Count = Entry.second; 319 // Get the callee name by branch target if it's a call branch 320 StringRef CalleeName = FunctionSamples::getCanonicalFnName( 321 Binary->getFuncFromStartOffset(TargetOffset)); 322 if (CalleeName.size() == 0) 323 continue; 324 325 // Record called target sample and its count 326 auto LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset); 327 if (!LeafLoc.hasValue()) 328 continue; 329 FunctionProfile.addCalledTargetSamples(LeafLoc->Callsite.LineOffset, 330 LeafLoc->Callsite.Discriminator, 331 CalleeName, Count); 332 333 // Record head sample for called target(callee) 334 SampleContextFrameVector CalleeCtx(ContextId.begin(), ContextId.end()); 335 assert(CalleeCtx.back().CallerName == LeafLoc->CallerName && 336 "Leaf function name doesn't match"); 337 CalleeCtx.back() = *LeafLoc; 338 CalleeCtx.emplace_back(CalleeName, LineLocation(0, 0)); 339 FunctionSamples &CalleeProfile = getFunctionProfileForContext(CalleeCtx); 340 assert(Count != 0 && "Unexpected zero weight branch"); 341 CalleeProfile.addHeadSamples(Count); 342 } 343 } 344 345 static SampleContextFrame 346 getCallerContext(SampleContextFrames CalleeContext, 347 SampleContextFrameVector &CallerContext) { 348 assert(CalleeContext.size() > 1 && "Unexpected empty context"); 349 CalleeContext = CalleeContext.drop_back(); 350 CallerContext.assign(CalleeContext.begin(), CalleeContext.end()); 351 SampleContextFrame CallerFrame = CallerContext.back(); 352 CallerContext.back().Callsite = LineLocation(0, 0); 353 return CallerFrame; 354 } 355 356 void CSProfileGenerator::populateInferredFunctionSamples() { 357 for (const auto &Item : ProfileMap) { 358 const auto &CalleeContext = Item.first; 359 const FunctionSamples &CalleeProfile = Item.second; 360 361 // If we already have head sample counts, we must have value profile 362 // for call sites added already. Skip to avoid double counting. 363 if (CalleeProfile.getHeadSamples()) 364 continue; 365 // If we don't have context, nothing to do for caller's call site. 366 // This could happen for entry point function. 367 if (CalleeContext.isBaseContext()) 368 continue; 369 370 // Infer Caller's frame loc and context ID through string splitting 371 SampleContextFrameVector CallerContextId; 372 SampleContextFrame &&CallerLeafFrameLoc = 373 getCallerContext(CalleeContext.getContextFrames(), CallerContextId); 374 SampleContextFrames CallerContext(CallerContextId); 375 376 // It's possible that we haven't seen any sample directly in the caller, 377 // in which case CallerProfile will not exist. But we can't modify 378 // ProfileMap while iterating it. 379 // TODO: created function profile for those callers too 380 if (ProfileMap.find(CallerContext) == ProfileMap.end()) 381 continue; 382 FunctionSamples &CallerProfile = ProfileMap[CallerContext]; 383 384 // Since we don't have call count for inlined functions, we 385 // estimate it from inlinee's profile using entry body sample. 386 uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples(); 387 // If we don't have samples with location, use 1 to indicate live. 388 if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size()) 389 EstimatedCallCount = 1; 390 CallerProfile.addCalledTargetSamples( 391 CallerLeafFrameLoc.Callsite.LineOffset, 392 CallerLeafFrameLoc.Callsite.Discriminator, 393 CalleeProfile.getContext().getName(), EstimatedCallCount); 394 CallerProfile.addBodySamples(CallerLeafFrameLoc.Callsite.LineOffset, 395 CallerLeafFrameLoc.Callsite.Discriminator, 396 EstimatedCallCount); 397 CallerProfile.addTotalSamples(EstimatedCallCount); 398 } 399 } 400 401 void CSProfileGenerator::postProcessProfiles() { 402 // Compute hot/cold threshold based on profile. This will be used for cold 403 // context profile merging/trimming. 404 computeSummaryAndThreshold(); 405 406 // Run global pre-inliner to adjust/merge context profile based on estimated 407 // inline decisions. 408 if (EnableCSPreInliner) { 409 CSPreInliner(ProfileMap, *Binary, HotCountThreshold, ColdCountThreshold) 410 .run(); 411 } 412 413 // Trim and merge cold context profile using cold threshold above. By default, 414 // we skip such merging and trimming when preinliner is on. 415 if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() || 416 CSProfMergeColdContext.getNumOccurrences()) { 417 SampleContextTrimmer(ProfileMap) 418 .trimAndMergeColdContextProfiles( 419 ColdCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, 420 CSProfMaxColdContextDepth); 421 } 422 } 423 424 void CSProfileGenerator::computeSummaryAndThreshold() { 425 // Update the default value of cold cutoff for llvm-profgen. 426 // Do it here because we don't want to change the global default, 427 // which would lead CS profile size too large. 428 if (!ProfileSummaryCutoffCold.getNumOccurrences()) 429 ProfileSummaryCutoffCold = 999000; 430 431 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); 432 auto Summary = Builder.computeSummaryForProfiles(ProfileMap); 433 HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold( 434 (Summary->getDetailedSummary())); 435 ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold( 436 (Summary->getDetailedSummary())); 437 } 438 439 void CSProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer, 440 SampleProfileMap &ProfileMap) { 441 if (std::error_code EC = Writer->write(ProfileMap)) 442 exitWithError(std::move(EC)); 443 } 444 445 // Helper function to extract context prefix string stack 446 // Extract context stack for reusing, leaf context stack will 447 // be added compressed while looking up function profile 448 static void extractPrefixContextStack( 449 SampleContextFrameVector &ContextStack, 450 const SmallVectorImpl<const MCDecodedPseudoProbe *> &Probes, 451 ProfiledBinary *Binary) { 452 for (const auto *P : Probes) { 453 Binary->getInlineContextForProbe(P, ContextStack, true); 454 } 455 } 456 457 void PseudoProbeCSProfileGenerator::generateProfile() { 458 // Enable pseudo probe functionalities in SampleProf 459 FunctionSamples::ProfileIsProbeBased = true; 460 FunctionSamples::ProfileIsCS = true; 461 for (const auto &CI : SampleCounters) { 462 const ProbeBasedCtxKey *CtxKey = 463 dyn_cast<ProbeBasedCtxKey>(CI.first.getPtr()); 464 SampleContextFrameVector ContextStack; 465 extractPrefixContextStack(ContextStack, CtxKey->Probes, Binary); 466 // Fill in function body samples from probes, also infer caller's samples 467 // from callee's probe 468 populateBodySamplesWithProbes(CI.second.RangeCounter, ContextStack); 469 // Fill in boundary samples for a call probe 470 populateBoundarySamplesWithProbes(CI.second.BranchCounter, ContextStack); 471 } 472 473 postProcessProfiles(); 474 } 475 476 void PseudoProbeCSProfileGenerator::extractProbesFromRange( 477 const RangeSample &RangeCounter, ProbeCounterMap &ProbeCounter) { 478 RangeSample Ranges; 479 findDisjointRanges(Ranges, RangeCounter); 480 for (const auto &Range : Ranges) { 481 uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); 482 uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); 483 uint64_t Count = Range.second; 484 // Disjoint ranges have introduce zero-filled gap that 485 // doesn't belong to current context, filter them out. 486 if (Count == 0) 487 continue; 488 489 InstructionPointer IP(Binary, RangeBegin, true); 490 491 // Disjoint ranges may have range in the middle of two instr, 492 // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range 493 // can be Addr1+1 to Addr2-1. We should ignore such range. 494 if (IP.Address > RangeEnd) 495 continue; 496 497 while (IP.Address <= RangeEnd) { 498 const AddressProbesMap &Address2ProbesMap = 499 Binary->getAddress2ProbesMap(); 500 auto It = Address2ProbesMap.find(IP.Address); 501 if (It != Address2ProbesMap.end()) { 502 for (const auto &Probe : It->second) { 503 if (!Probe.isBlock()) 504 continue; 505 ProbeCounter[&Probe] += Count; 506 } 507 } 508 509 IP.advance(); 510 } 511 } 512 } 513 514 void PseudoProbeCSProfileGenerator::populateBodySamplesWithProbes( 515 const RangeSample &RangeCounter, SampleContextFrames ContextStack) { 516 ProbeCounterMap ProbeCounter; 517 // Extract the top frame probes by looking up each address among the range in 518 // the Address2ProbeMap 519 extractProbesFromRange(RangeCounter, ProbeCounter); 520 std::unordered_map<MCDecodedPseudoProbeInlineTree *, 521 std::unordered_set<FunctionSamples *>> 522 FrameSamples; 523 for (auto PI : ProbeCounter) { 524 const MCDecodedPseudoProbe *Probe = PI.first; 525 uint64_t Count = PI.second; 526 FunctionSamples &FunctionProfile = 527 getFunctionProfileForLeafProbe(ContextStack, Probe); 528 // Record the current frame and FunctionProfile whenever samples are 529 // collected for non-danglie probes. This is for reporting all of the 530 // zero count probes of the frame later. 531 FrameSamples[Probe->getInlineTreeNode()].insert(&FunctionProfile); 532 FunctionProfile.addBodySamplesForProbe(Probe->getIndex(), Count); 533 FunctionProfile.addTotalSamples(Count); 534 if (Probe->isEntry()) { 535 FunctionProfile.addHeadSamples(Count); 536 // Look up for the caller's function profile 537 const auto *InlinerDesc = Binary->getInlinerDescForProbe(Probe); 538 if (InlinerDesc != nullptr) { 539 // Since the context id will be compressed, we have to use callee's 540 // context id to infer caller's context id to ensure they share the 541 // same context prefix. 542 SampleContextFrames CalleeContextId = 543 FunctionProfile.getContext().getContextFrames(); 544 SampleContextFrameVector CallerContextId; 545 SampleContextFrame &&CallerLeafFrameLoc = 546 getCallerContext(CalleeContextId, CallerContextId); 547 uint64_t CallerIndex = CallerLeafFrameLoc.Callsite.LineOffset; 548 assert(CallerIndex && 549 "Inferred caller's location index shouldn't be zero!"); 550 // Save the new context for future references. 551 SampleContextFrames CallerContext = 552 *Contexts.insert(CallerContextId).first; 553 FunctionSamples &CallerProfile = 554 getFunctionProfileForContext(CallerContext); 555 CallerProfile.setFunctionHash(InlinerDesc->FuncHash); 556 CallerProfile.addBodySamples(CallerIndex, 0, Count); 557 CallerProfile.addTotalSamples(Count); 558 CallerProfile.addCalledTargetSamples( 559 CallerIndex, 0, FunctionProfile.getContext().getName(), Count); 560 } 561 } 562 } 563 564 // Assign zero count for remaining probes without sample hits to 565 // differentiate from probes optimized away, of which the counts are unknown 566 // and will be inferred by the compiler. 567 for (auto &I : FrameSamples) { 568 for (auto *FunctionProfile : I.second) { 569 for (auto *Probe : I.first->getProbes()) { 570 FunctionProfile->addBodySamplesForProbe(Probe->getIndex(), 0); 571 } 572 } 573 } 574 } 575 576 void PseudoProbeCSProfileGenerator::populateBoundarySamplesWithProbes( 577 const BranchSample &BranchCounter, SampleContextFrames ContextStack) { 578 for (auto BI : BranchCounter) { 579 uint64_t SourceOffset = BI.first.first; 580 uint64_t TargetOffset = BI.first.second; 581 uint64_t Count = BI.second; 582 uint64_t SourceAddress = Binary->offsetToVirtualAddr(SourceOffset); 583 const MCDecodedPseudoProbe *CallProbe = 584 Binary->getCallProbeForAddr(SourceAddress); 585 if (CallProbe == nullptr) 586 continue; 587 FunctionSamples &FunctionProfile = 588 getFunctionProfileForLeafProbe(ContextStack, CallProbe); 589 FunctionProfile.addBodySamples(CallProbe->getIndex(), 0, Count); 590 FunctionProfile.addTotalSamples(Count); 591 StringRef CalleeName = FunctionSamples::getCanonicalFnName( 592 Binary->getFuncFromStartOffset(TargetOffset)); 593 if (CalleeName.size() == 0) 594 continue; 595 FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(), 0, CalleeName, 596 Count); 597 } 598 } 599 600 FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe( 601 SampleContextFrames ContextStack, const MCDecodedPseudoProbe *LeafProbe) { 602 603 // Explicitly copy the context for appending the leaf context 604 SampleContextFrameVector NewContextStack(ContextStack.begin(), 605 ContextStack.end()); 606 Binary->getInlineContextForProbe(LeafProbe, NewContextStack, true); 607 // For leaf inlined context with the top frame, we should strip off the top 608 // frame's probe id, like: 609 // Inlined stack: [foo:1, bar:2], the ContextId will be "foo:1 @ bar" 610 auto LeafFrame = NewContextStack.back(); 611 LeafFrame.Callsite = LineLocation(0, 0); 612 NewContextStack.pop_back(); 613 // Compress the context string except for the leaf frame 614 CSProfileGenerator::compressRecursionContext(NewContextStack); 615 CSProfileGenerator::trimContext(NewContextStack); 616 NewContextStack.push_back(LeafFrame); 617 // Save the new context for future references. 618 SampleContextFrames NewContext = *Contexts.insert(NewContextStack).first; 619 620 const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->getGuid()); 621 bool WasLeafInlined = LeafProbe->getInlineTreeNode()->hasInlineSite(); 622 FunctionSamples &FunctionProile = 623 getFunctionProfileForContext(NewContext, WasLeafInlined); 624 FunctionProile.setFunctionHash(FuncDesc->FuncHash); 625 return FunctionProile; 626 } 627 628 } // end namespace sampleprof 629 } // end namespace llvm 630