1de1ab26fSDiego Novillo //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===// 2de1ab26fSDiego Novillo // 3de1ab26fSDiego Novillo // The LLVM Compiler Infrastructure 4de1ab26fSDiego Novillo // 5de1ab26fSDiego Novillo // This file is distributed under the University of Illinois Open Source 6de1ab26fSDiego Novillo // License. See LICENSE.TXT for details. 7de1ab26fSDiego Novillo // 8de1ab26fSDiego Novillo //===----------------------------------------------------------------------===// 9de1ab26fSDiego Novillo // 10de1ab26fSDiego Novillo // This file implements the class that reads LLVM sample profiles. It 11bb5605caSDiego Novillo // supports three file formats: text, binary and gcov. 12de1ab26fSDiego Novillo // 13bb5605caSDiego Novillo // The textual representation is useful for debugging and testing purposes. The 14bb5605caSDiego Novillo // binary representation is more compact, resulting in smaller file sizes. 15de1ab26fSDiego Novillo // 16bb5605caSDiego Novillo // The gcov encoding is the one generated by GCC's AutoFDO profile creation 17bb5605caSDiego Novillo // tool (https://github.com/google/autofdo) 18de1ab26fSDiego Novillo // 19bb5605caSDiego Novillo // All three encodings can be used interchangeably as an input sample profile. 20de1ab26fSDiego Novillo // 21de1ab26fSDiego Novillo //===----------------------------------------------------------------------===// 22de1ab26fSDiego Novillo 23de1ab26fSDiego Novillo #include "llvm/ProfileData/SampleProfReader.h" 24b93483dbSDiego Novillo #include "llvm/ADT/DenseMap.h" 25b93483dbSDiego Novillo #include "llvm/ADT/SmallVector.h" 26de1ab26fSDiego Novillo #include "llvm/Support/Debug.h" 27de1ab26fSDiego Novillo #include "llvm/Support/ErrorOr.h" 28c572e92cSDiego Novillo #include "llvm/Support/LEB128.h" 29de1ab26fSDiego Novillo #include "llvm/Support/LineIterator.h" 30c572e92cSDiego Novillo #include "llvm/Support/MemoryBuffer.h" 31de1ab26fSDiego Novillo 32c572e92cSDiego Novillo using namespace llvm::sampleprof; 33de1ab26fSDiego Novillo using namespace llvm; 34de1ab26fSDiego Novillo 35de1ab26fSDiego Novillo /// \brief Dump the function profile for \p FName. 36de1ab26fSDiego Novillo /// 37de1ab26fSDiego Novillo /// \param FName Name of the function to print. 38d5336ae2SDiego Novillo /// \param OS Stream to emit the output to. 39d5336ae2SDiego Novillo void SampleProfileReader::dumpFunctionProfile(StringRef FName, 40d5336ae2SDiego Novillo raw_ostream &OS) { 418e415a82SDiego Novillo OS << "Function: " << FName << ": " << Profiles[FName]; 42de1ab26fSDiego Novillo } 43de1ab26fSDiego Novillo 44d5336ae2SDiego Novillo /// \brief Dump all the function profiles found on stream \p OS. 45d5336ae2SDiego Novillo void SampleProfileReader::dump(raw_ostream &OS) { 46d5336ae2SDiego Novillo for (const auto &I : Profiles) 47d5336ae2SDiego Novillo dumpFunctionProfile(I.getKey(), OS); 48de1ab26fSDiego Novillo } 49de1ab26fSDiego Novillo 506722688eSDehao Chen /// \brief Parse \p Input as function head. 516722688eSDehao Chen /// 526722688eSDehao Chen /// Parse one line of \p Input, and update function name in \p FName, 536722688eSDehao Chen /// function's total sample count in \p NumSamples, function's entry 546722688eSDehao Chen /// count in \p NumHeadSamples. 556722688eSDehao Chen /// 566722688eSDehao Chen /// \returns true if parsing is successful. 576722688eSDehao Chen static bool ParseHead(const StringRef &Input, StringRef &FName, 5838be3330SDiego Novillo uint64_t &NumSamples, uint64_t &NumHeadSamples) { 596722688eSDehao Chen if (Input[0] == ' ') 606722688eSDehao Chen return false; 616722688eSDehao Chen size_t n2 = Input.rfind(':'); 626722688eSDehao Chen size_t n1 = Input.rfind(':', n2 - 1); 636722688eSDehao Chen FName = Input.substr(0, n1); 646722688eSDehao Chen if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples)) 656722688eSDehao Chen return false; 666722688eSDehao Chen if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples)) 676722688eSDehao Chen return false; 686722688eSDehao Chen return true; 696722688eSDehao Chen } 706722688eSDehao Chen 7110042412SDehao Chen 7210042412SDehao Chen /// \brief Returns true if line offset \p L is legal (only has 16 bits). 7310042412SDehao Chen static bool isOffsetLegal(unsigned L) { 7410042412SDehao Chen return (L & 0xffff) == L; 7510042412SDehao Chen } 7610042412SDehao Chen 776722688eSDehao Chen /// \brief Parse \p Input as line sample. 786722688eSDehao Chen /// 796722688eSDehao Chen /// \param Input input line. 806722688eSDehao Chen /// \param IsCallsite true if the line represents an inlined callsite. 816722688eSDehao Chen /// \param Depth the depth of the inline stack. 826722688eSDehao Chen /// \param NumSamples total samples of the line/inlined callsite. 836722688eSDehao Chen /// \param LineOffset line offset to the start of the function. 846722688eSDehao Chen /// \param Discriminator discriminator of the line. 856722688eSDehao Chen /// \param TargetCountMap map from indirect call target to count. 866722688eSDehao Chen /// 876722688eSDehao Chen /// returns true if parsing is successful. 8838be3330SDiego Novillo static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, 8938be3330SDiego Novillo uint64_t &NumSamples, uint32_t &LineOffset, 9038be3330SDiego Novillo uint32_t &Discriminator, StringRef &CalleeName, 9138be3330SDiego Novillo DenseMap<StringRef, uint64_t> &TargetCountMap) { 926722688eSDehao Chen for (Depth = 0; Input[Depth] == ' '; Depth++) 936722688eSDehao Chen ; 946722688eSDehao Chen if (Depth == 0) 956722688eSDehao Chen return false; 966722688eSDehao Chen 976722688eSDehao Chen size_t n1 = Input.find(':'); 986722688eSDehao Chen StringRef Loc = Input.substr(Depth, n1 - Depth); 996722688eSDehao Chen size_t n2 = Loc.find('.'); 1006722688eSDehao Chen if (n2 == StringRef::npos) { 10110042412SDehao Chen if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset)) 1026722688eSDehao Chen return false; 1036722688eSDehao Chen Discriminator = 0; 1046722688eSDehao Chen } else { 1056722688eSDehao Chen if (Loc.substr(0, n2).getAsInteger(10, LineOffset)) 1066722688eSDehao Chen return false; 1076722688eSDehao Chen if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator)) 1086722688eSDehao Chen return false; 1096722688eSDehao Chen } 1106722688eSDehao Chen 1116722688eSDehao Chen StringRef Rest = Input.substr(n1 + 2); 1126722688eSDehao Chen if (Rest[0] >= '0' && Rest[0] <= '9') { 1136722688eSDehao Chen IsCallsite = false; 1146722688eSDehao Chen size_t n3 = Rest.find(' '); 1156722688eSDehao Chen if (n3 == StringRef::npos) { 1166722688eSDehao Chen if (Rest.getAsInteger(10, NumSamples)) 1176722688eSDehao Chen return false; 1186722688eSDehao Chen } else { 1196722688eSDehao Chen if (Rest.substr(0, n3).getAsInteger(10, NumSamples)) 1206722688eSDehao Chen return false; 1216722688eSDehao Chen } 1226722688eSDehao Chen while (n3 != StringRef::npos) { 1236722688eSDehao Chen n3 += Rest.substr(n3).find_first_not_of(' '); 1246722688eSDehao Chen Rest = Rest.substr(n3); 1256722688eSDehao Chen n3 = Rest.find(' '); 1266722688eSDehao Chen StringRef pair = Rest; 1276722688eSDehao Chen if (n3 != StringRef::npos) { 1286722688eSDehao Chen pair = Rest.substr(0, n3); 1296722688eSDehao Chen } 13038be3330SDiego Novillo size_t n4 = pair.find(':'); 13138be3330SDiego Novillo uint64_t count; 1326722688eSDehao Chen if (pair.substr(n4 + 1).getAsInteger(10, count)) 1336722688eSDehao Chen return false; 1346722688eSDehao Chen TargetCountMap[pair.substr(0, n4)] = count; 1356722688eSDehao Chen } 1366722688eSDehao Chen } else { 1376722688eSDehao Chen IsCallsite = true; 13838be3330SDiego Novillo size_t n3 = Rest.find_last_of(':'); 1396722688eSDehao Chen CalleeName = Rest.substr(0, n3); 1406722688eSDehao Chen if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) 1416722688eSDehao Chen return false; 1426722688eSDehao Chen } 1436722688eSDehao Chen return true; 1446722688eSDehao Chen } 1456722688eSDehao Chen 146de1ab26fSDiego Novillo /// \brief Load samples from a text file. 147de1ab26fSDiego Novillo /// 148de1ab26fSDiego Novillo /// See the documentation at the top of the file for an explanation of 149de1ab26fSDiego Novillo /// the expected format. 150de1ab26fSDiego Novillo /// 151de1ab26fSDiego Novillo /// \returns true if the file was loaded successfully, false otherwise. 152c572e92cSDiego Novillo std::error_code SampleProfileReaderText::read() { 153c572e92cSDiego Novillo line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); 154*48dd080cSNathan Slingerland sampleprof_error Result = sampleprof_error::success; 155de1ab26fSDiego Novillo 156aae1ed8eSDiego Novillo InlineCallStack InlineStack; 1576722688eSDehao Chen 1586722688eSDehao Chen for (; !LineIt.is_at_eof(); ++LineIt) { 1596722688eSDehao Chen if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') 1606722688eSDehao Chen continue; 161de1ab26fSDiego Novillo // Read the header of each function. 162de1ab26fSDiego Novillo // 163de1ab26fSDiego Novillo // Note that for function identifiers we are actually expecting 164de1ab26fSDiego Novillo // mangled names, but we may not always get them. This happens when 165de1ab26fSDiego Novillo // the compiler decides not to emit the function (e.g., it was inlined 166de1ab26fSDiego Novillo // and removed). In this case, the binary will not have the linkage 167de1ab26fSDiego Novillo // name for the function, so the profiler will emit the function's 168de1ab26fSDiego Novillo // unmangled name, which may contain characters like ':' and '>' in its 169de1ab26fSDiego Novillo // name (member functions, templates, etc). 170de1ab26fSDiego Novillo // 171de1ab26fSDiego Novillo // The only requirement we place on the identifier, then, is that it 172de1ab26fSDiego Novillo // should not begin with a number. 1736722688eSDehao Chen if ((*LineIt)[0] != ' ') { 17438be3330SDiego Novillo uint64_t NumSamples, NumHeadSamples; 1756722688eSDehao Chen StringRef FName; 1766722688eSDehao Chen if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) { 1773376a787SDiego Novillo reportError(LineIt.line_number(), 178de1ab26fSDiego Novillo "Expected 'mangled_name:NUM:NUM', found " + *LineIt); 179c572e92cSDiego Novillo return sampleprof_error::malformed; 180de1ab26fSDiego Novillo } 181de1ab26fSDiego Novillo Profiles[FName] = FunctionSamples(); 182de1ab26fSDiego Novillo FunctionSamples &FProfile = Profiles[FName]; 183*48dd080cSNathan Slingerland MergeResult(Result, FProfile.addTotalSamples(NumSamples)); 184*48dd080cSNathan Slingerland MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); 1856722688eSDehao Chen InlineStack.clear(); 1866722688eSDehao Chen InlineStack.push_back(&FProfile); 1876722688eSDehao Chen } else { 18838be3330SDiego Novillo uint64_t NumSamples; 1896722688eSDehao Chen StringRef FName; 19038be3330SDiego Novillo DenseMap<StringRef, uint64_t> TargetCountMap; 1916722688eSDehao Chen bool IsCallsite; 19238be3330SDiego Novillo uint32_t Depth, LineOffset, Discriminator; 1936722688eSDehao Chen if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset, 1946722688eSDehao Chen Discriminator, FName, TargetCountMap)) { 1953376a787SDiego Novillo reportError(LineIt.line_number(), 1963376a787SDiego Novillo "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + 1973376a787SDiego Novillo *LineIt); 198c572e92cSDiego Novillo return sampleprof_error::malformed; 199de1ab26fSDiego Novillo } 2006722688eSDehao Chen if (IsCallsite) { 2016722688eSDehao Chen while (InlineStack.size() > Depth) { 2026722688eSDehao Chen InlineStack.pop_back(); 203c572e92cSDiego Novillo } 2046722688eSDehao Chen FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( 2056722688eSDehao Chen CallsiteLocation(LineOffset, Discriminator, FName)); 206*48dd080cSNathan Slingerland MergeResult(Result, FSamples.addTotalSamples(NumSamples)); 2076722688eSDehao Chen InlineStack.push_back(&FSamples); 2086722688eSDehao Chen } else { 2096722688eSDehao Chen while (InlineStack.size() > Depth) { 2106722688eSDehao Chen InlineStack.pop_back(); 2116722688eSDehao Chen } 2126722688eSDehao Chen FunctionSamples &FProfile = *InlineStack.back(); 2136722688eSDehao Chen for (const auto &name_count : TargetCountMap) { 214*48dd080cSNathan Slingerland MergeResult(Result, FProfile.addCalledTargetSamples( 215*48dd080cSNathan Slingerland LineOffset, Discriminator, name_count.first, 216*48dd080cSNathan Slingerland name_count.second)); 217c572e92cSDiego Novillo } 218*48dd080cSNathan Slingerland MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, 219*48dd080cSNathan Slingerland NumSamples)); 2206722688eSDehao Chen } 221de1ab26fSDiego Novillo } 222de1ab26fSDiego Novillo } 223de1ab26fSDiego Novillo 224*48dd080cSNathan Slingerland return Result; 225de1ab26fSDiego Novillo } 226de1ab26fSDiego Novillo 2274f823667SNathan Slingerland bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { 2284f823667SNathan Slingerland bool result = false; 2294f823667SNathan Slingerland 2304f823667SNathan Slingerland // Check that the first non-comment line is a valid function header. 2314f823667SNathan Slingerland line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#'); 2324f823667SNathan Slingerland if (!LineIt.is_at_eof()) { 2334f823667SNathan Slingerland if ((*LineIt)[0] != ' ') { 2344f823667SNathan Slingerland uint64_t NumSamples, NumHeadSamples; 2354f823667SNathan Slingerland StringRef FName; 2364f823667SNathan Slingerland result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples); 2374f823667SNathan Slingerland } 2384f823667SNathan Slingerland } 2394f823667SNathan Slingerland 2404f823667SNathan Slingerland return result; 2414f823667SNathan Slingerland } 2424f823667SNathan Slingerland 243d5336ae2SDiego Novillo template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() { 244c572e92cSDiego Novillo unsigned NumBytesRead = 0; 245c572e92cSDiego Novillo std::error_code EC; 246c572e92cSDiego Novillo uint64_t Val = decodeULEB128(Data, &NumBytesRead); 247c572e92cSDiego Novillo 248c572e92cSDiego Novillo if (Val > std::numeric_limits<T>::max()) 249c572e92cSDiego Novillo EC = sampleprof_error::malformed; 250c572e92cSDiego Novillo else if (Data + NumBytesRead > End) 251c572e92cSDiego Novillo EC = sampleprof_error::truncated; 252c572e92cSDiego Novillo else 253c572e92cSDiego Novillo EC = sampleprof_error::success; 254c572e92cSDiego Novillo 255c572e92cSDiego Novillo if (EC) { 2563376a787SDiego Novillo reportError(0, EC.message()); 257c572e92cSDiego Novillo return EC; 258c572e92cSDiego Novillo } 259c572e92cSDiego Novillo 260c572e92cSDiego Novillo Data += NumBytesRead; 261c572e92cSDiego Novillo return static_cast<T>(Val); 262c572e92cSDiego Novillo } 263c572e92cSDiego Novillo 264c572e92cSDiego Novillo ErrorOr<StringRef> SampleProfileReaderBinary::readString() { 265c572e92cSDiego Novillo std::error_code EC; 266c572e92cSDiego Novillo StringRef Str(reinterpret_cast<const char *>(Data)); 267c572e92cSDiego Novillo if (Data + Str.size() + 1 > End) { 268c572e92cSDiego Novillo EC = sampleprof_error::truncated; 2693376a787SDiego Novillo reportError(0, EC.message()); 270c572e92cSDiego Novillo return EC; 271c572e92cSDiego Novillo } 272c572e92cSDiego Novillo 273c572e92cSDiego Novillo Data += Str.size() + 1; 274c572e92cSDiego Novillo return Str; 275c572e92cSDiego Novillo } 276c572e92cSDiego Novillo 277760c5a8fSDiego Novillo ErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() { 278760c5a8fSDiego Novillo std::error_code EC; 27938be3330SDiego Novillo auto Idx = readNumber<uint32_t>(); 280760c5a8fSDiego Novillo if (std::error_code EC = Idx.getError()) 281760c5a8fSDiego Novillo return EC; 282760c5a8fSDiego Novillo if (*Idx >= NameTable.size()) 283760c5a8fSDiego Novillo return sampleprof_error::truncated_name_table; 284760c5a8fSDiego Novillo return NameTable[*Idx]; 285760c5a8fSDiego Novillo } 286760c5a8fSDiego Novillo 287a7f1e8efSDiego Novillo std::error_code 288a7f1e8efSDiego Novillo SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { 289b93483dbSDiego Novillo auto NumSamples = readNumber<uint64_t>(); 290b93483dbSDiego Novillo if (std::error_code EC = NumSamples.getError()) 291c572e92cSDiego Novillo return EC; 292b93483dbSDiego Novillo FProfile.addTotalSamples(*NumSamples); 293c572e92cSDiego Novillo 294c572e92cSDiego Novillo // Read the samples in the body. 29538be3330SDiego Novillo auto NumRecords = readNumber<uint32_t>(); 296c572e92cSDiego Novillo if (std::error_code EC = NumRecords.getError()) 297c572e92cSDiego Novillo return EC; 298a7f1e8efSDiego Novillo 29938be3330SDiego Novillo for (uint32_t I = 0; I < *NumRecords; ++I) { 300c572e92cSDiego Novillo auto LineOffset = readNumber<uint64_t>(); 301c572e92cSDiego Novillo if (std::error_code EC = LineOffset.getError()) 302c572e92cSDiego Novillo return EC; 303c572e92cSDiego Novillo 30410042412SDehao Chen if (!isOffsetLegal(*LineOffset)) { 30510042412SDehao Chen return std::error_code(); 30610042412SDehao Chen } 30710042412SDehao Chen 308c572e92cSDiego Novillo auto Discriminator = readNumber<uint64_t>(); 309c572e92cSDiego Novillo if (std::error_code EC = Discriminator.getError()) 310c572e92cSDiego Novillo return EC; 311c572e92cSDiego Novillo 312c572e92cSDiego Novillo auto NumSamples = readNumber<uint64_t>(); 313c572e92cSDiego Novillo if (std::error_code EC = NumSamples.getError()) 314c572e92cSDiego Novillo return EC; 315c572e92cSDiego Novillo 31638be3330SDiego Novillo auto NumCalls = readNumber<uint32_t>(); 317c572e92cSDiego Novillo if (std::error_code EC = NumCalls.getError()) 318c572e92cSDiego Novillo return EC; 319c572e92cSDiego Novillo 32038be3330SDiego Novillo for (uint32_t J = 0; J < *NumCalls; ++J) { 321760c5a8fSDiego Novillo auto CalledFunction(readStringFromTable()); 322c572e92cSDiego Novillo if (std::error_code EC = CalledFunction.getError()) 323c572e92cSDiego Novillo return EC; 324c572e92cSDiego Novillo 325c572e92cSDiego Novillo auto CalledFunctionSamples = readNumber<uint64_t>(); 326c572e92cSDiego Novillo if (std::error_code EC = CalledFunctionSamples.getError()) 327c572e92cSDiego Novillo return EC; 328c572e92cSDiego Novillo 329c572e92cSDiego Novillo FProfile.addCalledTargetSamples(*LineOffset, *Discriminator, 330a7f1e8efSDiego Novillo *CalledFunction, *CalledFunctionSamples); 331c572e92cSDiego Novillo } 332c572e92cSDiego Novillo 333c572e92cSDiego Novillo FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples); 334c572e92cSDiego Novillo } 335a7f1e8efSDiego Novillo 336a7f1e8efSDiego Novillo // Read all the samples for inlined function calls. 33738be3330SDiego Novillo auto NumCallsites = readNumber<uint32_t>(); 338a7f1e8efSDiego Novillo if (std::error_code EC = NumCallsites.getError()) 339a7f1e8efSDiego Novillo return EC; 340a7f1e8efSDiego Novillo 34138be3330SDiego Novillo for (uint32_t J = 0; J < *NumCallsites; ++J) { 342a7f1e8efSDiego Novillo auto LineOffset = readNumber<uint64_t>(); 343a7f1e8efSDiego Novillo if (std::error_code EC = LineOffset.getError()) 344a7f1e8efSDiego Novillo return EC; 345a7f1e8efSDiego Novillo 346a7f1e8efSDiego Novillo auto Discriminator = readNumber<uint64_t>(); 347a7f1e8efSDiego Novillo if (std::error_code EC = Discriminator.getError()) 348a7f1e8efSDiego Novillo return EC; 349a7f1e8efSDiego Novillo 350760c5a8fSDiego Novillo auto FName(readStringFromTable()); 351a7f1e8efSDiego Novillo if (std::error_code EC = FName.getError()) 352a7f1e8efSDiego Novillo return EC; 353a7f1e8efSDiego Novillo 354a7f1e8efSDiego Novillo FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( 355a7f1e8efSDiego Novillo CallsiteLocation(*LineOffset, *Discriminator, *FName)); 356a7f1e8efSDiego Novillo if (std::error_code EC = readProfile(CalleeProfile)) 357a7f1e8efSDiego Novillo return EC; 358a7f1e8efSDiego Novillo } 359a7f1e8efSDiego Novillo 360a7f1e8efSDiego Novillo return sampleprof_error::success; 361a7f1e8efSDiego Novillo } 362a7f1e8efSDiego Novillo 363a7f1e8efSDiego Novillo std::error_code SampleProfileReaderBinary::read() { 364a7f1e8efSDiego Novillo while (!at_eof()) { 365b93483dbSDiego Novillo auto NumHeadSamples = readNumber<uint64_t>(); 366b93483dbSDiego Novillo if (std::error_code EC = NumHeadSamples.getError()) 367b93483dbSDiego Novillo return EC; 368b93483dbSDiego Novillo 369760c5a8fSDiego Novillo auto FName(readStringFromTable()); 370a7f1e8efSDiego Novillo if (std::error_code EC = FName.getError()) 371a7f1e8efSDiego Novillo return EC; 372a7f1e8efSDiego Novillo 373a7f1e8efSDiego Novillo Profiles[*FName] = FunctionSamples(); 374a7f1e8efSDiego Novillo FunctionSamples &FProfile = Profiles[*FName]; 375a7f1e8efSDiego Novillo 376b93483dbSDiego Novillo FProfile.addHeadSamples(*NumHeadSamples); 377b93483dbSDiego Novillo 378a7f1e8efSDiego Novillo if (std::error_code EC = readProfile(FProfile)) 379a7f1e8efSDiego Novillo return EC; 380c572e92cSDiego Novillo } 381c572e92cSDiego Novillo 382c572e92cSDiego Novillo return sampleprof_error::success; 383c572e92cSDiego Novillo } 384c572e92cSDiego Novillo 385c572e92cSDiego Novillo std::error_code SampleProfileReaderBinary::readHeader() { 386c572e92cSDiego Novillo Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); 387c572e92cSDiego Novillo End = Data + Buffer->getBufferSize(); 388c572e92cSDiego Novillo 389c572e92cSDiego Novillo // Read and check the magic identifier. 390c572e92cSDiego Novillo auto Magic = readNumber<uint64_t>(); 391c572e92cSDiego Novillo if (std::error_code EC = Magic.getError()) 392c572e92cSDiego Novillo return EC; 393c572e92cSDiego Novillo else if (*Magic != SPMagic()) 394c572e92cSDiego Novillo return sampleprof_error::bad_magic; 395c572e92cSDiego Novillo 396c572e92cSDiego Novillo // Read the version number. 397c572e92cSDiego Novillo auto Version = readNumber<uint64_t>(); 398c572e92cSDiego Novillo if (std::error_code EC = Version.getError()) 399c572e92cSDiego Novillo return EC; 400c572e92cSDiego Novillo else if (*Version != SPVersion()) 401c572e92cSDiego Novillo return sampleprof_error::unsupported_version; 402c572e92cSDiego Novillo 403760c5a8fSDiego Novillo // Read the name table. 40438be3330SDiego Novillo auto Size = readNumber<uint32_t>(); 405760c5a8fSDiego Novillo if (std::error_code EC = Size.getError()) 406760c5a8fSDiego Novillo return EC; 407760c5a8fSDiego Novillo NameTable.reserve(*Size); 40838be3330SDiego Novillo for (uint32_t I = 0; I < *Size; ++I) { 409760c5a8fSDiego Novillo auto Name(readString()); 410760c5a8fSDiego Novillo if (std::error_code EC = Name.getError()) 411760c5a8fSDiego Novillo return EC; 412760c5a8fSDiego Novillo NameTable.push_back(*Name); 413760c5a8fSDiego Novillo } 414760c5a8fSDiego Novillo 415c572e92cSDiego Novillo return sampleprof_error::success; 416c572e92cSDiego Novillo } 417c572e92cSDiego Novillo 418c572e92cSDiego Novillo bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) { 419c572e92cSDiego Novillo const uint8_t *Data = 420c572e92cSDiego Novillo reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 421c572e92cSDiego Novillo uint64_t Magic = decodeULEB128(Data); 422c572e92cSDiego Novillo return Magic == SPMagic(); 423c572e92cSDiego Novillo } 424c572e92cSDiego Novillo 4253376a787SDiego Novillo std::error_code SampleProfileReaderGCC::skipNextWord() { 4263376a787SDiego Novillo uint32_t dummy; 4273376a787SDiego Novillo if (!GcovBuffer.readInt(dummy)) 4283376a787SDiego Novillo return sampleprof_error::truncated; 4293376a787SDiego Novillo return sampleprof_error::success; 4303376a787SDiego Novillo } 4313376a787SDiego Novillo 4323376a787SDiego Novillo template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() { 4333376a787SDiego Novillo if (sizeof(T) <= sizeof(uint32_t)) { 4343376a787SDiego Novillo uint32_t Val; 4353376a787SDiego Novillo if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max()) 4363376a787SDiego Novillo return static_cast<T>(Val); 4373376a787SDiego Novillo } else if (sizeof(T) <= sizeof(uint64_t)) { 4383376a787SDiego Novillo uint64_t Val; 4393376a787SDiego Novillo if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max()) 4403376a787SDiego Novillo return static_cast<T>(Val); 4413376a787SDiego Novillo } 4423376a787SDiego Novillo 4433376a787SDiego Novillo std::error_code EC = sampleprof_error::malformed; 4443376a787SDiego Novillo reportError(0, EC.message()); 4453376a787SDiego Novillo return EC; 4463376a787SDiego Novillo } 4473376a787SDiego Novillo 4483376a787SDiego Novillo ErrorOr<StringRef> SampleProfileReaderGCC::readString() { 4493376a787SDiego Novillo StringRef Str; 4503376a787SDiego Novillo if (!GcovBuffer.readString(Str)) 4513376a787SDiego Novillo return sampleprof_error::truncated; 4523376a787SDiego Novillo return Str; 4533376a787SDiego Novillo } 4543376a787SDiego Novillo 4553376a787SDiego Novillo std::error_code SampleProfileReaderGCC::readHeader() { 4563376a787SDiego Novillo // Read the magic identifier. 4573376a787SDiego Novillo if (!GcovBuffer.readGCDAFormat()) 4583376a787SDiego Novillo return sampleprof_error::unrecognized_format; 4593376a787SDiego Novillo 4603376a787SDiego Novillo // Read the version number. Note - the GCC reader does not validate this 4613376a787SDiego Novillo // version, but the profile creator generates v704. 4623376a787SDiego Novillo GCOV::GCOVVersion version; 4633376a787SDiego Novillo if (!GcovBuffer.readGCOVVersion(version)) 4643376a787SDiego Novillo return sampleprof_error::unrecognized_format; 4653376a787SDiego Novillo 4663376a787SDiego Novillo if (version != GCOV::V704) 4673376a787SDiego Novillo return sampleprof_error::unsupported_version; 4683376a787SDiego Novillo 4693376a787SDiego Novillo // Skip the empty integer. 4703376a787SDiego Novillo if (std::error_code EC = skipNextWord()) 4713376a787SDiego Novillo return EC; 4723376a787SDiego Novillo 4733376a787SDiego Novillo return sampleprof_error::success; 4743376a787SDiego Novillo } 4753376a787SDiego Novillo 4763376a787SDiego Novillo std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) { 4773376a787SDiego Novillo uint32_t Tag; 4783376a787SDiego Novillo if (!GcovBuffer.readInt(Tag)) 4793376a787SDiego Novillo return sampleprof_error::truncated; 4803376a787SDiego Novillo 4813376a787SDiego Novillo if (Tag != Expected) 4823376a787SDiego Novillo return sampleprof_error::malformed; 4833376a787SDiego Novillo 4843376a787SDiego Novillo if (std::error_code EC = skipNextWord()) 4853376a787SDiego Novillo return EC; 4863376a787SDiego Novillo 4873376a787SDiego Novillo return sampleprof_error::success; 4883376a787SDiego Novillo } 4893376a787SDiego Novillo 4903376a787SDiego Novillo std::error_code SampleProfileReaderGCC::readNameTable() { 4913376a787SDiego Novillo if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames)) 4923376a787SDiego Novillo return EC; 4933376a787SDiego Novillo 4943376a787SDiego Novillo uint32_t Size; 4953376a787SDiego Novillo if (!GcovBuffer.readInt(Size)) 4963376a787SDiego Novillo return sampleprof_error::truncated; 4973376a787SDiego Novillo 4983376a787SDiego Novillo for (uint32_t I = 0; I < Size; ++I) { 4993376a787SDiego Novillo StringRef Str; 5003376a787SDiego Novillo if (!GcovBuffer.readString(Str)) 5013376a787SDiego Novillo return sampleprof_error::truncated; 5023376a787SDiego Novillo Names.push_back(Str); 5033376a787SDiego Novillo } 5043376a787SDiego Novillo 5053376a787SDiego Novillo return sampleprof_error::success; 5063376a787SDiego Novillo } 5073376a787SDiego Novillo 5083376a787SDiego Novillo std::error_code SampleProfileReaderGCC::readFunctionProfiles() { 5093376a787SDiego Novillo if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction)) 5103376a787SDiego Novillo return EC; 5113376a787SDiego Novillo 5123376a787SDiego Novillo uint32_t NumFunctions; 5133376a787SDiego Novillo if (!GcovBuffer.readInt(NumFunctions)) 5143376a787SDiego Novillo return sampleprof_error::truncated; 5153376a787SDiego Novillo 516aae1ed8eSDiego Novillo InlineCallStack Stack; 5173376a787SDiego Novillo for (uint32_t I = 0; I < NumFunctions; ++I) 518aae1ed8eSDiego Novillo if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) 5193376a787SDiego Novillo return EC; 5203376a787SDiego Novillo 5213376a787SDiego Novillo return sampleprof_error::success; 5223376a787SDiego Novillo } 5233376a787SDiego Novillo 524aae1ed8eSDiego Novillo std::error_code SampleProfileReaderGCC::readOneFunctionProfile( 525aae1ed8eSDiego Novillo const InlineCallStack &InlineStack, bool Update, uint32_t Offset) { 5263376a787SDiego Novillo uint64_t HeadCount = 0; 527aae1ed8eSDiego Novillo if (InlineStack.size() == 0) 5283376a787SDiego Novillo if (!GcovBuffer.readInt64(HeadCount)) 5293376a787SDiego Novillo return sampleprof_error::truncated; 5303376a787SDiego Novillo 5313376a787SDiego Novillo uint32_t NameIdx; 5323376a787SDiego Novillo if (!GcovBuffer.readInt(NameIdx)) 5333376a787SDiego Novillo return sampleprof_error::truncated; 5343376a787SDiego Novillo 5353376a787SDiego Novillo StringRef Name(Names[NameIdx]); 5363376a787SDiego Novillo 5373376a787SDiego Novillo uint32_t NumPosCounts; 5383376a787SDiego Novillo if (!GcovBuffer.readInt(NumPosCounts)) 5393376a787SDiego Novillo return sampleprof_error::truncated; 5403376a787SDiego Novillo 541aae1ed8eSDiego Novillo uint32_t NumCallsites; 542aae1ed8eSDiego Novillo if (!GcovBuffer.readInt(NumCallsites)) 5433376a787SDiego Novillo return sampleprof_error::truncated; 5443376a787SDiego Novillo 545aae1ed8eSDiego Novillo FunctionSamples *FProfile = nullptr; 546aae1ed8eSDiego Novillo if (InlineStack.size() == 0) { 547aae1ed8eSDiego Novillo // If this is a top function that we have already processed, do not 548aae1ed8eSDiego Novillo // update its profile again. This happens in the presence of 549aae1ed8eSDiego Novillo // function aliases. Since these aliases share the same function 550aae1ed8eSDiego Novillo // body, there will be identical replicated profiles for the 551aae1ed8eSDiego Novillo // original function. In this case, we simply not bother updating 552aae1ed8eSDiego Novillo // the profile of the original function. 553aae1ed8eSDiego Novillo FProfile = &Profiles[Name]; 554aae1ed8eSDiego Novillo FProfile->addHeadSamples(HeadCount); 555aae1ed8eSDiego Novillo if (FProfile->getTotalSamples() > 0) 5563376a787SDiego Novillo Update = false; 557aae1ed8eSDiego Novillo } else { 558aae1ed8eSDiego Novillo // Otherwise, we are reading an inlined instance. The top of the 559aae1ed8eSDiego Novillo // inline stack contains the profile of the caller. Insert this 560aae1ed8eSDiego Novillo // callee in the caller's CallsiteMap. 561aae1ed8eSDiego Novillo FunctionSamples *CallerProfile = InlineStack.front(); 562aae1ed8eSDiego Novillo uint32_t LineOffset = Offset >> 16; 563aae1ed8eSDiego Novillo uint32_t Discriminator = Offset & 0xffff; 564aae1ed8eSDiego Novillo FProfile = &CallerProfile->functionSamplesAt( 565aae1ed8eSDiego Novillo CallsiteLocation(LineOffset, Discriminator, Name)); 5663376a787SDiego Novillo } 5673376a787SDiego Novillo 5683376a787SDiego Novillo for (uint32_t I = 0; I < NumPosCounts; ++I) { 5693376a787SDiego Novillo uint32_t Offset; 5703376a787SDiego Novillo if (!GcovBuffer.readInt(Offset)) 5713376a787SDiego Novillo return sampleprof_error::truncated; 5723376a787SDiego Novillo 5733376a787SDiego Novillo uint32_t NumTargets; 5743376a787SDiego Novillo if (!GcovBuffer.readInt(NumTargets)) 5753376a787SDiego Novillo return sampleprof_error::truncated; 5763376a787SDiego Novillo 5773376a787SDiego Novillo uint64_t Count; 5783376a787SDiego Novillo if (!GcovBuffer.readInt64(Count)) 5793376a787SDiego Novillo return sampleprof_error::truncated; 5803376a787SDiego Novillo 581aae1ed8eSDiego Novillo // The line location is encoded in the offset as: 582aae1ed8eSDiego Novillo // high 16 bits: line offset to the start of the function. 583aae1ed8eSDiego Novillo // low 16 bits: discriminator. 584aae1ed8eSDiego Novillo uint32_t LineOffset = Offset >> 16; 585aae1ed8eSDiego Novillo uint32_t Discriminator = Offset & 0xffff; 5863376a787SDiego Novillo 587aae1ed8eSDiego Novillo InlineCallStack NewStack; 588aae1ed8eSDiego Novillo NewStack.push_back(FProfile); 589aae1ed8eSDiego Novillo NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 590aae1ed8eSDiego Novillo if (Update) { 591aae1ed8eSDiego Novillo // Walk up the inline stack, adding the samples on this line to 592aae1ed8eSDiego Novillo // the total sample count of the callers in the chain. 593aae1ed8eSDiego Novillo for (auto CallerProfile : NewStack) 594aae1ed8eSDiego Novillo CallerProfile->addTotalSamples(Count); 595aae1ed8eSDiego Novillo 596aae1ed8eSDiego Novillo // Update the body samples for the current profile. 597aae1ed8eSDiego Novillo FProfile->addBodySamples(LineOffset, Discriminator, Count); 598aae1ed8eSDiego Novillo } 599aae1ed8eSDiego Novillo 600aae1ed8eSDiego Novillo // Process the list of functions called at an indirect call site. 601aae1ed8eSDiego Novillo // These are all the targets that a function pointer (or virtual 602aae1ed8eSDiego Novillo // function) resolved at runtime. 6033376a787SDiego Novillo for (uint32_t J = 0; J < NumTargets; J++) { 6043376a787SDiego Novillo uint32_t HistVal; 6053376a787SDiego Novillo if (!GcovBuffer.readInt(HistVal)) 6063376a787SDiego Novillo return sampleprof_error::truncated; 6073376a787SDiego Novillo 6083376a787SDiego Novillo if (HistVal != HIST_TYPE_INDIR_CALL_TOPN) 6093376a787SDiego Novillo return sampleprof_error::malformed; 6103376a787SDiego Novillo 6113376a787SDiego Novillo uint64_t TargetIdx; 6123376a787SDiego Novillo if (!GcovBuffer.readInt64(TargetIdx)) 6133376a787SDiego Novillo return sampleprof_error::truncated; 6143376a787SDiego Novillo StringRef TargetName(Names[TargetIdx]); 6153376a787SDiego Novillo 6163376a787SDiego Novillo uint64_t TargetCount; 6173376a787SDiego Novillo if (!GcovBuffer.readInt64(TargetCount)) 6183376a787SDiego Novillo return sampleprof_error::truncated; 6193376a787SDiego Novillo 6203376a787SDiego Novillo if (Update) { 6213376a787SDiego Novillo FunctionSamples &TargetProfile = Profiles[TargetName]; 622aae1ed8eSDiego Novillo TargetProfile.addCalledTargetSamples(LineOffset, Discriminator, 623aae1ed8eSDiego Novillo TargetName, TargetCount); 6243376a787SDiego Novillo } 6253376a787SDiego Novillo } 6263376a787SDiego Novillo } 6273376a787SDiego Novillo 628aae1ed8eSDiego Novillo // Process all the inlined callers into the current function. These 629aae1ed8eSDiego Novillo // are all the callsites that were inlined into this function. 630aae1ed8eSDiego Novillo for (uint32_t I = 0; I < NumCallsites; I++) { 6313376a787SDiego Novillo // The offset is encoded as: 6323376a787SDiego Novillo // high 16 bits: line offset to the start of the function. 6333376a787SDiego Novillo // low 16 bits: discriminator. 6343376a787SDiego Novillo uint32_t Offset; 6353376a787SDiego Novillo if (!GcovBuffer.readInt(Offset)) 6363376a787SDiego Novillo return sampleprof_error::truncated; 637aae1ed8eSDiego Novillo InlineCallStack NewStack; 638aae1ed8eSDiego Novillo NewStack.push_back(FProfile); 639aae1ed8eSDiego Novillo NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 640aae1ed8eSDiego Novillo if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset)) 6413376a787SDiego Novillo return EC; 6423376a787SDiego Novillo } 6433376a787SDiego Novillo 6443376a787SDiego Novillo return sampleprof_error::success; 6453376a787SDiego Novillo } 6463376a787SDiego Novillo 6473376a787SDiego Novillo /// \brief Read a GCC AutoFDO profile. 6483376a787SDiego Novillo /// 6493376a787SDiego Novillo /// This format is generated by the Linux Perf conversion tool at 6503376a787SDiego Novillo /// https://github.com/google/autofdo. 6513376a787SDiego Novillo std::error_code SampleProfileReaderGCC::read() { 6523376a787SDiego Novillo // Read the string table. 6533376a787SDiego Novillo if (std::error_code EC = readNameTable()) 6543376a787SDiego Novillo return EC; 6553376a787SDiego Novillo 6563376a787SDiego Novillo // Read the source profile. 6573376a787SDiego Novillo if (std::error_code EC = readFunctionProfiles()) 6583376a787SDiego Novillo return EC; 6593376a787SDiego Novillo 6603376a787SDiego Novillo return sampleprof_error::success; 6613376a787SDiego Novillo } 6623376a787SDiego Novillo 6633376a787SDiego Novillo bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { 6643376a787SDiego Novillo StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart())); 6653376a787SDiego Novillo return Magic == "adcg*704"; 6663376a787SDiego Novillo } 6673376a787SDiego Novillo 668c572e92cSDiego Novillo /// \brief Prepare a memory buffer for the contents of \p Filename. 669de1ab26fSDiego Novillo /// 670c572e92cSDiego Novillo /// \returns an error code indicating the status of the buffer. 671fcd55607SDiego Novillo static ErrorOr<std::unique_ptr<MemoryBuffer>> 672fcd55607SDiego Novillo setupMemoryBuffer(std::string Filename) { 673c572e92cSDiego Novillo auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); 674c572e92cSDiego Novillo if (std::error_code EC = BufferOrErr.getError()) 675c572e92cSDiego Novillo return EC; 676fcd55607SDiego Novillo auto Buffer = std::move(BufferOrErr.get()); 677c572e92cSDiego Novillo 678c572e92cSDiego Novillo // Sanity check the file. 67938be3330SDiego Novillo if (Buffer->getBufferSize() > std::numeric_limits<uint32_t>::max()) 680c572e92cSDiego Novillo return sampleprof_error::too_large; 681c572e92cSDiego Novillo 682fcd55607SDiego Novillo return std::move(Buffer); 683c572e92cSDiego Novillo } 684c572e92cSDiego Novillo 685c572e92cSDiego Novillo /// \brief Create a sample profile reader based on the format of the input file. 686c572e92cSDiego Novillo /// 687c572e92cSDiego Novillo /// \param Filename The file to open. 688c572e92cSDiego Novillo /// 689c572e92cSDiego Novillo /// \param Reader The reader to instantiate according to \p Filename's format. 690c572e92cSDiego Novillo /// 691c572e92cSDiego Novillo /// \param C The LLVM context to use to emit diagnostics. 692c572e92cSDiego Novillo /// 693c572e92cSDiego Novillo /// \returns an error code indicating the status of the created reader. 694fcd55607SDiego Novillo ErrorOr<std::unique_ptr<SampleProfileReader>> 695fcd55607SDiego Novillo SampleProfileReader::create(StringRef Filename, LLVMContext &C) { 696fcd55607SDiego Novillo auto BufferOrError = setupMemoryBuffer(Filename); 697fcd55607SDiego Novillo if (std::error_code EC = BufferOrError.getError()) 698c572e92cSDiego Novillo return EC; 69951abea74SNathan Slingerland return create(BufferOrError.get(), C); 70051abea74SNathan Slingerland } 701c572e92cSDiego Novillo 70251abea74SNathan Slingerland /// \brief Create a sample profile reader based on the format of the input data. 70351abea74SNathan Slingerland /// 70451abea74SNathan Slingerland /// \param B The memory buffer to create the reader from (assumes ownership). 70551abea74SNathan Slingerland /// 70651abea74SNathan Slingerland /// \param Reader The reader to instantiate according to \p Filename's format. 70751abea74SNathan Slingerland /// 70851abea74SNathan Slingerland /// \param C The LLVM context to use to emit diagnostics. 70951abea74SNathan Slingerland /// 71051abea74SNathan Slingerland /// \returns an error code indicating the status of the created reader. 71151abea74SNathan Slingerland ErrorOr<std::unique_ptr<SampleProfileReader>> 71251abea74SNathan Slingerland SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) { 713fcd55607SDiego Novillo std::unique_ptr<SampleProfileReader> Reader; 71451abea74SNathan Slingerland if (SampleProfileReaderBinary::hasFormat(*B)) 71551abea74SNathan Slingerland Reader.reset(new SampleProfileReaderBinary(std::move(B), C)); 71651abea74SNathan Slingerland else if (SampleProfileReaderGCC::hasFormat(*B)) 71751abea74SNathan Slingerland Reader.reset(new SampleProfileReaderGCC(std::move(B), C)); 71851abea74SNathan Slingerland else if (SampleProfileReaderText::hasFormat(*B)) 71951abea74SNathan Slingerland Reader.reset(new SampleProfileReaderText(std::move(B), C)); 7204f823667SNathan Slingerland else 7214f823667SNathan Slingerland return sampleprof_error::unrecognized_format; 722c572e92cSDiego Novillo 723fcd55607SDiego Novillo if (std::error_code EC = Reader->readHeader()) 724fcd55607SDiego Novillo return EC; 725fcd55607SDiego Novillo 726fcd55607SDiego Novillo return std::move(Reader); 727de1ab26fSDiego Novillo } 728