1*0b57cec5SDimitry Andric //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===// 2*0b57cec5SDimitry Andric // 3*0b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*0b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5*0b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*0b57cec5SDimitry Andric // 7*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 8*0b57cec5SDimitry Andric // 9*0b57cec5SDimitry Andric // This file contains support for writing profiling data for clang's 10*0b57cec5SDimitry Andric // instrumentation based PGO and coverage. 11*0b57cec5SDimitry Andric // 12*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 13*0b57cec5SDimitry Andric 14*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfWriter.h" 15*0b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 16*0b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h" 17*0b57cec5SDimitry Andric #include "llvm/IR/ProfileSummary.h" 18*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProf.h" 1981ad6265SDimitry Andric #include "llvm/ProfileData/MemProf.h" 20*0b57cec5SDimitry Andric #include "llvm/ProfileData/ProfileCommon.h" 21*0b57cec5SDimitry Andric #include "llvm/Support/Endian.h" 22*0b57cec5SDimitry Andric #include "llvm/Support/EndianStream.h" 23*0b57cec5SDimitry Andric #include "llvm/Support/Error.h" 24*0b57cec5SDimitry Andric #include "llvm/Support/MemoryBuffer.h" 25*0b57cec5SDimitry Andric #include "llvm/Support/OnDiskHashTable.h" 26*0b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h" 27*0b57cec5SDimitry Andric #include <cstdint> 28*0b57cec5SDimitry Andric #include <memory> 29*0b57cec5SDimitry Andric #include <string> 30*0b57cec5SDimitry Andric #include <tuple> 31*0b57cec5SDimitry Andric #include <utility> 32*0b57cec5SDimitry Andric #include <vector> 33*0b57cec5SDimitry Andric 34*0b57cec5SDimitry Andric using namespace llvm; 35*0b57cec5SDimitry Andric 36*0b57cec5SDimitry Andric // A struct to define how the data stream should be patched. For Indexed 37*0b57cec5SDimitry Andric // profiling, only uint64_t data type is needed. 38*0b57cec5SDimitry Andric struct PatchItem { 39*0b57cec5SDimitry Andric uint64_t Pos; // Where to patch. 40*0b57cec5SDimitry Andric uint64_t *D; // Pointer to an array of source data. 41*0b57cec5SDimitry Andric int N; // Number of elements in \c D array. 42*0b57cec5SDimitry Andric }; 43*0b57cec5SDimitry Andric 44*0b57cec5SDimitry Andric namespace llvm { 45*0b57cec5SDimitry Andric 46*0b57cec5SDimitry Andric // A wrapper class to abstract writer stream with support of bytes 47*0b57cec5SDimitry Andric // back patching. 48*0b57cec5SDimitry Andric class ProfOStream { 49*0b57cec5SDimitry Andric public: 50*0b57cec5SDimitry Andric ProfOStream(raw_fd_ostream &FD) 51*0b57cec5SDimitry Andric : IsFDOStream(true), OS(FD), LE(FD, support::little) {} 52*0b57cec5SDimitry Andric ProfOStream(raw_string_ostream &STR) 53*0b57cec5SDimitry Andric : IsFDOStream(false), OS(STR), LE(STR, support::little) {} 54*0b57cec5SDimitry Andric 55*0b57cec5SDimitry Andric uint64_t tell() { return OS.tell(); } 56*0b57cec5SDimitry Andric void write(uint64_t V) { LE.write<uint64_t>(V); } 57*0b57cec5SDimitry Andric 58*0b57cec5SDimitry Andric // \c patch can only be called when all data is written and flushed. 59*0b57cec5SDimitry Andric // For raw_string_ostream, the patch is done on the target string 60*0b57cec5SDimitry Andric // directly and it won't be reflected in the stream's internal buffer. 61*0b57cec5SDimitry Andric void patch(PatchItem *P, int NItems) { 62*0b57cec5SDimitry Andric using namespace support; 63*0b57cec5SDimitry Andric 64*0b57cec5SDimitry Andric if (IsFDOStream) { 65*0b57cec5SDimitry Andric raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS); 6681ad6265SDimitry Andric const uint64_t LastPos = FDOStream.tell(); 67*0b57cec5SDimitry Andric for (int K = 0; K < NItems; K++) { 68*0b57cec5SDimitry Andric FDOStream.seek(P[K].Pos); 69*0b57cec5SDimitry Andric for (int I = 0; I < P[K].N; I++) 70*0b57cec5SDimitry Andric write(P[K].D[I]); 71*0b57cec5SDimitry Andric } 7281ad6265SDimitry Andric // Reset the stream to the last position after patching so that users 7381ad6265SDimitry Andric // don't accidentally overwrite data. This makes it consistent with 7481ad6265SDimitry Andric // the string stream below which replaces the data directly. 7581ad6265SDimitry Andric FDOStream.seek(LastPos); 76*0b57cec5SDimitry Andric } else { 77*0b57cec5SDimitry Andric raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS); 78*0b57cec5SDimitry Andric std::string &Data = SOStream.str(); // with flush 79*0b57cec5SDimitry Andric for (int K = 0; K < NItems; K++) { 80*0b57cec5SDimitry Andric for (int I = 0; I < P[K].N; I++) { 81*0b57cec5SDimitry Andric uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]); 82*0b57cec5SDimitry Andric Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t), 83*0b57cec5SDimitry Andric (const char *)&Bytes, sizeof(uint64_t)); 84*0b57cec5SDimitry Andric } 85*0b57cec5SDimitry Andric } 86*0b57cec5SDimitry Andric } 87*0b57cec5SDimitry Andric } 88*0b57cec5SDimitry Andric 89*0b57cec5SDimitry Andric // If \c OS is an instance of \c raw_fd_ostream, this field will be 90*0b57cec5SDimitry Andric // true. Otherwise, \c OS will be an raw_string_ostream. 91*0b57cec5SDimitry Andric bool IsFDOStream; 92*0b57cec5SDimitry Andric raw_ostream &OS; 93*0b57cec5SDimitry Andric support::endian::Writer LE; 94*0b57cec5SDimitry Andric }; 95*0b57cec5SDimitry Andric 96*0b57cec5SDimitry Andric class InstrProfRecordWriterTrait { 97*0b57cec5SDimitry Andric public: 98*0b57cec5SDimitry Andric using key_type = StringRef; 99*0b57cec5SDimitry Andric using key_type_ref = StringRef; 100*0b57cec5SDimitry Andric 101*0b57cec5SDimitry Andric using data_type = const InstrProfWriter::ProfilingData *const; 102*0b57cec5SDimitry Andric using data_type_ref = const InstrProfWriter::ProfilingData *const; 103*0b57cec5SDimitry Andric 104*0b57cec5SDimitry Andric using hash_value_type = uint64_t; 105*0b57cec5SDimitry Andric using offset_type = uint64_t; 106*0b57cec5SDimitry Andric 107*0b57cec5SDimitry Andric support::endianness ValueProfDataEndianness = support::little; 108*0b57cec5SDimitry Andric InstrProfSummaryBuilder *SummaryBuilder; 109*0b57cec5SDimitry Andric InstrProfSummaryBuilder *CSSummaryBuilder; 110*0b57cec5SDimitry Andric 111*0b57cec5SDimitry Andric InstrProfRecordWriterTrait() = default; 112*0b57cec5SDimitry Andric 113*0b57cec5SDimitry Andric static hash_value_type ComputeHash(key_type_ref K) { 114*0b57cec5SDimitry Andric return IndexedInstrProf::ComputeHash(K); 115*0b57cec5SDimitry Andric } 116*0b57cec5SDimitry Andric 117*0b57cec5SDimitry Andric static std::pair<offset_type, offset_type> 118*0b57cec5SDimitry Andric EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { 119*0b57cec5SDimitry Andric using namespace support; 120*0b57cec5SDimitry Andric 121*0b57cec5SDimitry Andric endian::Writer LE(Out, little); 122*0b57cec5SDimitry Andric 123*0b57cec5SDimitry Andric offset_type N = K.size(); 124*0b57cec5SDimitry Andric LE.write<offset_type>(N); 125*0b57cec5SDimitry Andric 126*0b57cec5SDimitry Andric offset_type M = 0; 127*0b57cec5SDimitry Andric for (const auto &ProfileData : *V) { 128*0b57cec5SDimitry Andric const InstrProfRecord &ProfRecord = ProfileData.second; 129*0b57cec5SDimitry Andric M += sizeof(uint64_t); // The function hash 130*0b57cec5SDimitry Andric M += sizeof(uint64_t); // The size of the Counts vector 131*0b57cec5SDimitry Andric M += ProfRecord.Counts.size() * sizeof(uint64_t); 132*0b57cec5SDimitry Andric 133*0b57cec5SDimitry Andric // Value data 134*0b57cec5SDimitry Andric M += ValueProfData::getSize(ProfileData.second); 135*0b57cec5SDimitry Andric } 136*0b57cec5SDimitry Andric LE.write<offset_type>(M); 137*0b57cec5SDimitry Andric 138*0b57cec5SDimitry Andric return std::make_pair(N, M); 139*0b57cec5SDimitry Andric } 140*0b57cec5SDimitry Andric 141*0b57cec5SDimitry Andric void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) { 142*0b57cec5SDimitry Andric Out.write(K.data(), N); 143*0b57cec5SDimitry Andric } 144*0b57cec5SDimitry Andric 145*0b57cec5SDimitry Andric void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) { 146*0b57cec5SDimitry Andric using namespace support; 147*0b57cec5SDimitry Andric 148*0b57cec5SDimitry Andric endian::Writer LE(Out, little); 149*0b57cec5SDimitry Andric for (const auto &ProfileData : *V) { 150*0b57cec5SDimitry Andric const InstrProfRecord &ProfRecord = ProfileData.second; 151*0b57cec5SDimitry Andric if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first)) 152*0b57cec5SDimitry Andric CSSummaryBuilder->addRecord(ProfRecord); 153*0b57cec5SDimitry Andric else 154*0b57cec5SDimitry Andric SummaryBuilder->addRecord(ProfRecord); 155*0b57cec5SDimitry Andric 156*0b57cec5SDimitry Andric LE.write<uint64_t>(ProfileData.first); // Function hash 157*0b57cec5SDimitry Andric LE.write<uint64_t>(ProfRecord.Counts.size()); 158*0b57cec5SDimitry Andric for (uint64_t I : ProfRecord.Counts) 159*0b57cec5SDimitry Andric LE.write<uint64_t>(I); 160*0b57cec5SDimitry Andric 161*0b57cec5SDimitry Andric // Write value data 162*0b57cec5SDimitry Andric std::unique_ptr<ValueProfData> VDataPtr = 163*0b57cec5SDimitry Andric ValueProfData::serializeFrom(ProfileData.second); 164*0b57cec5SDimitry Andric uint32_t S = VDataPtr->getSize(); 165*0b57cec5SDimitry Andric VDataPtr->swapBytesFromHost(ValueProfDataEndianness); 166*0b57cec5SDimitry Andric Out.write((const char *)VDataPtr.get(), S); 167*0b57cec5SDimitry Andric } 168*0b57cec5SDimitry Andric } 169*0b57cec5SDimitry Andric }; 170*0b57cec5SDimitry Andric 171*0b57cec5SDimitry Andric } // end namespace llvm 172*0b57cec5SDimitry Andric 1731fd87a68SDimitry Andric InstrProfWriter::InstrProfWriter(bool Sparse) 1741fd87a68SDimitry Andric : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {} 175*0b57cec5SDimitry Andric 176*0b57cec5SDimitry Andric InstrProfWriter::~InstrProfWriter() { delete InfoObj; } 177*0b57cec5SDimitry Andric 178*0b57cec5SDimitry Andric // Internal interface for testing purpose only. 179*0b57cec5SDimitry Andric void InstrProfWriter::setValueProfDataEndianness( 180*0b57cec5SDimitry Andric support::endianness Endianness) { 181*0b57cec5SDimitry Andric InfoObj->ValueProfDataEndianness = Endianness; 182*0b57cec5SDimitry Andric } 183*0b57cec5SDimitry Andric 184*0b57cec5SDimitry Andric void InstrProfWriter::setOutputSparse(bool Sparse) { 185*0b57cec5SDimitry Andric this->Sparse = Sparse; 186*0b57cec5SDimitry Andric } 187*0b57cec5SDimitry Andric 188*0b57cec5SDimitry Andric void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight, 189*0b57cec5SDimitry Andric function_ref<void(Error)> Warn) { 190*0b57cec5SDimitry Andric auto Name = I.Name; 191*0b57cec5SDimitry Andric auto Hash = I.Hash; 192*0b57cec5SDimitry Andric addRecord(Name, Hash, std::move(I), Weight, Warn); 193*0b57cec5SDimitry Andric } 194*0b57cec5SDimitry Andric 195*0b57cec5SDimitry Andric void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other, 196*0b57cec5SDimitry Andric OverlapStats &Overlap, 197*0b57cec5SDimitry Andric OverlapStats &FuncLevelOverlap, 198*0b57cec5SDimitry Andric const OverlapFuncFilters &FuncFilter) { 199*0b57cec5SDimitry Andric auto Name = Other.Name; 200*0b57cec5SDimitry Andric auto Hash = Other.Hash; 2018bcb0991SDimitry Andric Other.accumulateCounts(FuncLevelOverlap.Test); 202*0b57cec5SDimitry Andric if (FunctionData.find(Name) == FunctionData.end()) { 203*0b57cec5SDimitry Andric Overlap.addOneUnique(FuncLevelOverlap.Test); 204*0b57cec5SDimitry Andric return; 205*0b57cec5SDimitry Andric } 206*0b57cec5SDimitry Andric if (FuncLevelOverlap.Test.CountSum < 1.0f) { 207*0b57cec5SDimitry Andric Overlap.Overlap.NumEntries += 1; 208*0b57cec5SDimitry Andric return; 209*0b57cec5SDimitry Andric } 210*0b57cec5SDimitry Andric auto &ProfileDataMap = FunctionData[Name]; 211*0b57cec5SDimitry Andric bool NewFunc; 212*0b57cec5SDimitry Andric ProfilingData::iterator Where; 213*0b57cec5SDimitry Andric std::tie(Where, NewFunc) = 214*0b57cec5SDimitry Andric ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord())); 215*0b57cec5SDimitry Andric if (NewFunc) { 216*0b57cec5SDimitry Andric Overlap.addOneMismatch(FuncLevelOverlap.Test); 217*0b57cec5SDimitry Andric return; 218*0b57cec5SDimitry Andric } 219*0b57cec5SDimitry Andric InstrProfRecord &Dest = Where->second; 220*0b57cec5SDimitry Andric 221*0b57cec5SDimitry Andric uint64_t ValueCutoff = FuncFilter.ValueCutoff; 222349cc55cSDimitry Andric if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter)) 223*0b57cec5SDimitry Andric ValueCutoff = 0; 224*0b57cec5SDimitry Andric 225*0b57cec5SDimitry Andric Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff); 226*0b57cec5SDimitry Andric } 227*0b57cec5SDimitry Andric 228*0b57cec5SDimitry Andric void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash, 229*0b57cec5SDimitry Andric InstrProfRecord &&I, uint64_t Weight, 230*0b57cec5SDimitry Andric function_ref<void(Error)> Warn) { 231*0b57cec5SDimitry Andric auto &ProfileDataMap = FunctionData[Name]; 232*0b57cec5SDimitry Andric 233*0b57cec5SDimitry Andric bool NewFunc; 234*0b57cec5SDimitry Andric ProfilingData::iterator Where; 235*0b57cec5SDimitry Andric std::tie(Where, NewFunc) = 236*0b57cec5SDimitry Andric ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord())); 237*0b57cec5SDimitry Andric InstrProfRecord &Dest = Where->second; 238*0b57cec5SDimitry Andric 239*0b57cec5SDimitry Andric auto MapWarn = [&](instrprof_error E) { 240*0b57cec5SDimitry Andric Warn(make_error<InstrProfError>(E)); 241*0b57cec5SDimitry Andric }; 242*0b57cec5SDimitry Andric 243*0b57cec5SDimitry Andric if (NewFunc) { 244*0b57cec5SDimitry Andric // We've never seen a function with this name and hash, add it. 245*0b57cec5SDimitry Andric Dest = std::move(I); 246*0b57cec5SDimitry Andric if (Weight > 1) 247e8d8bef9SDimitry Andric Dest.scale(Weight, 1, MapWarn); 248*0b57cec5SDimitry Andric } else { 249*0b57cec5SDimitry Andric // We're updating a function we've seen before. 250*0b57cec5SDimitry Andric Dest.merge(I, Weight, MapWarn); 251*0b57cec5SDimitry Andric } 252*0b57cec5SDimitry Andric 253*0b57cec5SDimitry Andric Dest.sortValueData(); 254*0b57cec5SDimitry Andric } 255*0b57cec5SDimitry Andric 25681ad6265SDimitry Andric void InstrProfWriter::addMemProfRecord( 25781ad6265SDimitry Andric const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) { 25881ad6265SDimitry Andric auto Result = MemProfRecordData.insert({Id, Record}); 25981ad6265SDimitry Andric // If we inserted a new record then we are done. 26081ad6265SDimitry Andric if (Result.second) { 26181ad6265SDimitry Andric return; 26281ad6265SDimitry Andric } 26381ad6265SDimitry Andric memprof::IndexedMemProfRecord &Existing = Result.first->second; 26481ad6265SDimitry Andric Existing.merge(Record); 26581ad6265SDimitry Andric } 26681ad6265SDimitry Andric 26781ad6265SDimitry Andric bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id, 26881ad6265SDimitry Andric const memprof::Frame &Frame, 26981ad6265SDimitry Andric function_ref<void(Error)> Warn) { 27081ad6265SDimitry Andric auto Result = MemProfFrameData.insert({Id, Frame}); 27181ad6265SDimitry Andric // If a mapping already exists for the current frame id and it does not 27281ad6265SDimitry Andric // match the new mapping provided then reset the existing contents and bail 27381ad6265SDimitry Andric // out. We don't support the merging of memprof data whose Frame -> Id 27481ad6265SDimitry Andric // mapping across profiles is inconsistent. 27581ad6265SDimitry Andric if (!Result.second && Result.first->second != Frame) { 27681ad6265SDimitry Andric Warn(make_error<InstrProfError>(instrprof_error::malformed, 27781ad6265SDimitry Andric "frame to id mapping mismatch")); 27881ad6265SDimitry Andric return false; 27981ad6265SDimitry Andric } 28081ad6265SDimitry Andric return true; 28181ad6265SDimitry Andric } 28281ad6265SDimitry Andric 283*0b57cec5SDimitry Andric void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW, 284*0b57cec5SDimitry Andric function_ref<void(Error)> Warn) { 285*0b57cec5SDimitry Andric for (auto &I : IPW.FunctionData) 286*0b57cec5SDimitry Andric for (auto &Func : I.getValue()) 287*0b57cec5SDimitry Andric addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn); 28881ad6265SDimitry Andric 28981ad6265SDimitry Andric MemProfFrameData.reserve(IPW.MemProfFrameData.size()); 29081ad6265SDimitry Andric for (auto &I : IPW.MemProfFrameData) { 29181ad6265SDimitry Andric // If we weren't able to add the frame mappings then it doesn't make sense 29281ad6265SDimitry Andric // to try to merge the records from this profile. 29381ad6265SDimitry Andric if (!addMemProfFrame(I.first, I.second, Warn)) 29481ad6265SDimitry Andric return; 29581ad6265SDimitry Andric } 29681ad6265SDimitry Andric 29781ad6265SDimitry Andric MemProfRecordData.reserve(IPW.MemProfRecordData.size()); 29881ad6265SDimitry Andric for (auto &I : IPW.MemProfRecordData) { 29981ad6265SDimitry Andric addMemProfRecord(I.first, I.second); 30081ad6265SDimitry Andric } 301*0b57cec5SDimitry Andric } 302*0b57cec5SDimitry Andric 303*0b57cec5SDimitry Andric bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { 304*0b57cec5SDimitry Andric if (!Sparse) 305*0b57cec5SDimitry Andric return true; 306*0b57cec5SDimitry Andric for (const auto &Func : PD) { 307*0b57cec5SDimitry Andric const InstrProfRecord &IPR = Func.second; 308*0b57cec5SDimitry Andric if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) 309*0b57cec5SDimitry Andric return true; 310*0b57cec5SDimitry Andric } 311*0b57cec5SDimitry Andric return false; 312*0b57cec5SDimitry Andric } 313*0b57cec5SDimitry Andric 314*0b57cec5SDimitry Andric static void setSummary(IndexedInstrProf::Summary *TheSummary, 315*0b57cec5SDimitry Andric ProfileSummary &PS) { 316*0b57cec5SDimitry Andric using namespace IndexedInstrProf; 317*0b57cec5SDimitry Andric 318349cc55cSDimitry Andric const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary(); 319*0b57cec5SDimitry Andric TheSummary->NumSummaryFields = Summary::NumKinds; 320*0b57cec5SDimitry Andric TheSummary->NumCutoffEntries = Res.size(); 321*0b57cec5SDimitry Andric TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount()); 322*0b57cec5SDimitry Andric TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount()); 323*0b57cec5SDimitry Andric TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount()); 324*0b57cec5SDimitry Andric TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount()); 325*0b57cec5SDimitry Andric TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts()); 326*0b57cec5SDimitry Andric TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions()); 327*0b57cec5SDimitry Andric for (unsigned I = 0; I < Res.size(); I++) 328*0b57cec5SDimitry Andric TheSummary->setEntry(I, Res[I]); 329*0b57cec5SDimitry Andric } 330*0b57cec5SDimitry Andric 331fe6060f1SDimitry Andric Error InstrProfWriter::writeImpl(ProfOStream &OS) { 332*0b57cec5SDimitry Andric using namespace IndexedInstrProf; 333*0b57cec5SDimitry Andric 334*0b57cec5SDimitry Andric OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator; 335*0b57cec5SDimitry Andric 336*0b57cec5SDimitry Andric InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs); 337*0b57cec5SDimitry Andric InfoObj->SummaryBuilder = &ISB; 338*0b57cec5SDimitry Andric InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs); 339*0b57cec5SDimitry Andric InfoObj->CSSummaryBuilder = &CSISB; 340*0b57cec5SDimitry Andric 341*0b57cec5SDimitry Andric // Populate the hash table generator. 342*0b57cec5SDimitry Andric for (const auto &I : FunctionData) 343*0b57cec5SDimitry Andric if (shouldEncodeData(I.getValue())) 344*0b57cec5SDimitry Andric Generator.insert(I.getKey(), &I.getValue()); 34581ad6265SDimitry Andric 346*0b57cec5SDimitry Andric // Write the header. 347*0b57cec5SDimitry Andric IndexedInstrProf::Header Header; 348*0b57cec5SDimitry Andric Header.Magic = IndexedInstrProf::Magic; 349*0b57cec5SDimitry Andric Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion; 35081ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation)) 351*0b57cec5SDimitry Andric Header.Version |= VARIANT_MASK_IR_PROF; 35281ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) 353*0b57cec5SDimitry Andric Header.Version |= VARIANT_MASK_CSIR_PROF; 35481ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & 35581ad6265SDimitry Andric InstrProfKind::FunctionEntryInstrumentation)) 356e8d8bef9SDimitry Andric Header.Version |= VARIANT_MASK_INSTR_ENTRY; 3571fd87a68SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage)) 3581fd87a68SDimitry Andric Header.Version |= VARIANT_MASK_BYTE_COVERAGE; 3591fd87a68SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly)) 3601fd87a68SDimitry Andric Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY; 36181ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) 36281ad6265SDimitry Andric Header.Version |= VARIANT_MASK_MEMPROF; 363e8d8bef9SDimitry Andric 364*0b57cec5SDimitry Andric Header.Unused = 0; 365*0b57cec5SDimitry Andric Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType); 366*0b57cec5SDimitry Andric Header.HashOffset = 0; 36781ad6265SDimitry Andric Header.MemProfOffset = 0; 368*0b57cec5SDimitry Andric int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); 369*0b57cec5SDimitry Andric 37081ad6265SDimitry Andric // Only write out all the fields except 'HashOffset' and 'MemProfOffset'. We 37181ad6265SDimitry Andric // need to remember the offset of these fields to allow back patching later. 37281ad6265SDimitry Andric for (int I = 0; I < N - 2; I++) 373*0b57cec5SDimitry Andric OS.write(reinterpret_cast<uint64_t *>(&Header)[I]); 374*0b57cec5SDimitry Andric 375*0b57cec5SDimitry Andric // Save the location of Header.HashOffset field in \c OS. 376*0b57cec5SDimitry Andric uint64_t HashTableStartFieldOffset = OS.tell(); 377*0b57cec5SDimitry Andric // Reserve the space for HashOffset field. 378*0b57cec5SDimitry Andric OS.write(0); 379*0b57cec5SDimitry Andric 38081ad6265SDimitry Andric // Save the location of MemProf profile data. This is stored in two parts as 38181ad6265SDimitry Andric // the schema and as a separate on-disk chained hashtable. 38281ad6265SDimitry Andric uint64_t MemProfSectionOffset = OS.tell(); 38381ad6265SDimitry Andric // Reserve space for the MemProf table field to be patched later if this 38481ad6265SDimitry Andric // profile contains memory profile information. 38581ad6265SDimitry Andric OS.write(0); 38681ad6265SDimitry Andric 387*0b57cec5SDimitry Andric // Reserve space to write profile summary data. 388*0b57cec5SDimitry Andric uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); 389*0b57cec5SDimitry Andric uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); 390*0b57cec5SDimitry Andric // Remember the summary offset. 391*0b57cec5SDimitry Andric uint64_t SummaryOffset = OS.tell(); 392*0b57cec5SDimitry Andric for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) 393*0b57cec5SDimitry Andric OS.write(0); 394*0b57cec5SDimitry Andric uint64_t CSSummaryOffset = 0; 395*0b57cec5SDimitry Andric uint64_t CSSummarySize = 0; 39681ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) { 397*0b57cec5SDimitry Andric CSSummaryOffset = OS.tell(); 398*0b57cec5SDimitry Andric CSSummarySize = SummarySize / sizeof(uint64_t); 399*0b57cec5SDimitry Andric for (unsigned I = 0; I < CSSummarySize; I++) 400*0b57cec5SDimitry Andric OS.write(0); 401*0b57cec5SDimitry Andric } 402*0b57cec5SDimitry Andric 403*0b57cec5SDimitry Andric // Write the hash table. 404*0b57cec5SDimitry Andric uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj); 405*0b57cec5SDimitry Andric 40681ad6265SDimitry Andric // Write the MemProf profile data if we have it. This includes a simple schema 40781ad6265SDimitry Andric // with the format described below followed by the hashtable: 40881ad6265SDimitry Andric // uint64_t RecordTableOffset = RecordTableGenerator.Emit 40981ad6265SDimitry Andric // uint64_t FramePayloadOffset = Stream offset before emitting the frame table 41081ad6265SDimitry Andric // uint64_t FrameTableOffset = FrameTableGenerator.Emit 41181ad6265SDimitry Andric // uint64_t Num schema entries 41281ad6265SDimitry Andric // uint64_t Schema entry 0 41381ad6265SDimitry Andric // uint64_t Schema entry 1 41481ad6265SDimitry Andric // .... 41581ad6265SDimitry Andric // uint64_t Schema entry N - 1 41681ad6265SDimitry Andric // OnDiskChainedHashTable MemProfRecordData 41781ad6265SDimitry Andric // OnDiskChainedHashTable MemProfFrameData 41881ad6265SDimitry Andric uint64_t MemProfSectionStart = 0; 41981ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) { 42081ad6265SDimitry Andric MemProfSectionStart = OS.tell(); 42181ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof record table offset. 42281ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof frame payload offset. 42381ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof frame table offset. 42481ad6265SDimitry Andric 42581ad6265SDimitry Andric auto Schema = memprof::PortableMemInfoBlock::getSchema(); 42681ad6265SDimitry Andric OS.write(static_cast<uint64_t>(Schema.size())); 42781ad6265SDimitry Andric for (const auto Id : Schema) { 42881ad6265SDimitry Andric OS.write(static_cast<uint64_t>(Id)); 42981ad6265SDimitry Andric } 43081ad6265SDimitry Andric 43181ad6265SDimitry Andric auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>(); 43281ad6265SDimitry Andric RecordWriter->Schema = &Schema; 43381ad6265SDimitry Andric OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait> 43481ad6265SDimitry Andric RecordTableGenerator; 43581ad6265SDimitry Andric for (auto &I : MemProfRecordData) { 43681ad6265SDimitry Andric // Insert the key (func hash) and value (memprof record). 43781ad6265SDimitry Andric RecordTableGenerator.insert(I.first, I.second); 43881ad6265SDimitry Andric } 43981ad6265SDimitry Andric 44081ad6265SDimitry Andric uint64_t RecordTableOffset = 44181ad6265SDimitry Andric RecordTableGenerator.Emit(OS.OS, *RecordWriter); 44281ad6265SDimitry Andric 44381ad6265SDimitry Andric uint64_t FramePayloadOffset = OS.tell(); 44481ad6265SDimitry Andric 44581ad6265SDimitry Andric auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>(); 44681ad6265SDimitry Andric OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait> 44781ad6265SDimitry Andric FrameTableGenerator; 44881ad6265SDimitry Andric for (auto &I : MemProfFrameData) { 44981ad6265SDimitry Andric // Insert the key (frame id) and value (frame contents). 45081ad6265SDimitry Andric FrameTableGenerator.insert(I.first, I.second); 45181ad6265SDimitry Andric } 45281ad6265SDimitry Andric 45381ad6265SDimitry Andric uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter); 45481ad6265SDimitry Andric 45581ad6265SDimitry Andric PatchItem PatchItems[] = { 45681ad6265SDimitry Andric {MemProfSectionStart, &RecordTableOffset, 1}, 45781ad6265SDimitry Andric {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1}, 45881ad6265SDimitry Andric {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1}, 45981ad6265SDimitry Andric }; 46081ad6265SDimitry Andric OS.patch(PatchItems, 3); 46181ad6265SDimitry Andric } 46281ad6265SDimitry Andric 463*0b57cec5SDimitry Andric // Allocate space for data to be serialized out. 464*0b57cec5SDimitry Andric std::unique_ptr<IndexedInstrProf::Summary> TheSummary = 465*0b57cec5SDimitry Andric IndexedInstrProf::allocSummary(SummarySize); 466*0b57cec5SDimitry Andric // Compute the Summary and copy the data to the data 467*0b57cec5SDimitry Andric // structure to be serialized out (to disk or buffer). 468*0b57cec5SDimitry Andric std::unique_ptr<ProfileSummary> PS = ISB.getSummary(); 469*0b57cec5SDimitry Andric setSummary(TheSummary.get(), *PS); 470*0b57cec5SDimitry Andric InfoObj->SummaryBuilder = nullptr; 471*0b57cec5SDimitry Andric 472*0b57cec5SDimitry Andric // For Context Sensitive summary. 473*0b57cec5SDimitry Andric std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr; 47481ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) { 475*0b57cec5SDimitry Andric TheCSSummary = IndexedInstrProf::allocSummary(SummarySize); 476*0b57cec5SDimitry Andric std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary(); 477*0b57cec5SDimitry Andric setSummary(TheCSSummary.get(), *CSPS); 478*0b57cec5SDimitry Andric } 479*0b57cec5SDimitry Andric InfoObj->CSSummaryBuilder = nullptr; 480*0b57cec5SDimitry Andric 481*0b57cec5SDimitry Andric // Now do the final patch: 482*0b57cec5SDimitry Andric PatchItem PatchItems[] = { 483*0b57cec5SDimitry Andric // Patch the Header.HashOffset field. 484*0b57cec5SDimitry Andric {HashTableStartFieldOffset, &HashTableStart, 1}, 48581ad6265SDimitry Andric // Patch the Header.MemProfOffset (=0 for profiles without MemProf data). 48681ad6265SDimitry Andric {MemProfSectionOffset, &MemProfSectionStart, 1}, 487*0b57cec5SDimitry Andric // Patch the summary data. 488*0b57cec5SDimitry Andric {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()), 489*0b57cec5SDimitry Andric (int)(SummarySize / sizeof(uint64_t))}, 490*0b57cec5SDimitry Andric {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()), 491*0b57cec5SDimitry Andric (int)CSSummarySize}}; 492*0b57cec5SDimitry Andric 493*0b57cec5SDimitry Andric OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems)); 494fe6060f1SDimitry Andric 495fe6060f1SDimitry Andric for (const auto &I : FunctionData) 496fe6060f1SDimitry Andric for (const auto &F : I.getValue()) 497fe6060f1SDimitry Andric if (Error E = validateRecord(F.second)) 498fe6060f1SDimitry Andric return E; 499fe6060f1SDimitry Andric 500fe6060f1SDimitry Andric return Error::success(); 501*0b57cec5SDimitry Andric } 502*0b57cec5SDimitry Andric 503fe6060f1SDimitry Andric Error InstrProfWriter::write(raw_fd_ostream &OS) { 504*0b57cec5SDimitry Andric // Write the hash table. 505*0b57cec5SDimitry Andric ProfOStream POS(OS); 506fe6060f1SDimitry Andric return writeImpl(POS); 507*0b57cec5SDimitry Andric } 508*0b57cec5SDimitry Andric 509*0b57cec5SDimitry Andric std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() { 510*0b57cec5SDimitry Andric std::string Data; 511*0b57cec5SDimitry Andric raw_string_ostream OS(Data); 512*0b57cec5SDimitry Andric ProfOStream POS(OS); 513*0b57cec5SDimitry Andric // Write the hash table. 514fe6060f1SDimitry Andric if (Error E = writeImpl(POS)) 515fe6060f1SDimitry Andric return nullptr; 516*0b57cec5SDimitry Andric // Return this in an aligned memory buffer. 517*0b57cec5SDimitry Andric return MemoryBuffer::getMemBufferCopy(Data); 518*0b57cec5SDimitry Andric } 519*0b57cec5SDimitry Andric 520*0b57cec5SDimitry Andric static const char *ValueProfKindStr[] = { 521*0b57cec5SDimitry Andric #define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator, 522*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc" 523*0b57cec5SDimitry Andric }; 524*0b57cec5SDimitry Andric 525fe6060f1SDimitry Andric Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) { 526fe6060f1SDimitry Andric for (uint32_t VK = 0; VK <= IPVK_Last; VK++) { 527fe6060f1SDimitry Andric uint32_t NS = Func.getNumValueSites(VK); 528fe6060f1SDimitry Andric if (!NS) 529fe6060f1SDimitry Andric continue; 530fe6060f1SDimitry Andric for (uint32_t S = 0; S < NS; S++) { 531fe6060f1SDimitry Andric uint32_t ND = Func.getNumValueDataForSite(VK, S); 532fe6060f1SDimitry Andric std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S); 533fe6060f1SDimitry Andric bool WasZero = false; 534fe6060f1SDimitry Andric for (uint32_t I = 0; I < ND; I++) 535fe6060f1SDimitry Andric if ((VK != IPVK_IndirectCallTarget) && (VD[I].Value == 0)) { 536fe6060f1SDimitry Andric if (WasZero) 537fe6060f1SDimitry Andric return make_error<InstrProfError>(instrprof_error::invalid_prof); 538fe6060f1SDimitry Andric WasZero = true; 539fe6060f1SDimitry Andric } 540fe6060f1SDimitry Andric } 541fe6060f1SDimitry Andric } 542fe6060f1SDimitry Andric 543fe6060f1SDimitry Andric return Error::success(); 544fe6060f1SDimitry Andric } 545fe6060f1SDimitry Andric 546*0b57cec5SDimitry Andric void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, 547*0b57cec5SDimitry Andric const InstrProfRecord &Func, 548*0b57cec5SDimitry Andric InstrProfSymtab &Symtab, 549*0b57cec5SDimitry Andric raw_fd_ostream &OS) { 550*0b57cec5SDimitry Andric OS << Name << "\n"; 551*0b57cec5SDimitry Andric OS << "# Func Hash:\n" << Hash << "\n"; 552*0b57cec5SDimitry Andric OS << "# Num Counters:\n" << Func.Counts.size() << "\n"; 553*0b57cec5SDimitry Andric OS << "# Counter Values:\n"; 554*0b57cec5SDimitry Andric for (uint64_t Count : Func.Counts) 555*0b57cec5SDimitry Andric OS << Count << "\n"; 556*0b57cec5SDimitry Andric 557*0b57cec5SDimitry Andric uint32_t NumValueKinds = Func.getNumValueKinds(); 558*0b57cec5SDimitry Andric if (!NumValueKinds) { 559*0b57cec5SDimitry Andric OS << "\n"; 560*0b57cec5SDimitry Andric return; 561*0b57cec5SDimitry Andric } 562*0b57cec5SDimitry Andric 563*0b57cec5SDimitry Andric OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n"; 564*0b57cec5SDimitry Andric for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) { 565*0b57cec5SDimitry Andric uint32_t NS = Func.getNumValueSites(VK); 566*0b57cec5SDimitry Andric if (!NS) 567*0b57cec5SDimitry Andric continue; 568*0b57cec5SDimitry Andric OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n"; 569*0b57cec5SDimitry Andric OS << "# NumValueSites:\n" << NS << "\n"; 570*0b57cec5SDimitry Andric for (uint32_t S = 0; S < NS; S++) { 571*0b57cec5SDimitry Andric uint32_t ND = Func.getNumValueDataForSite(VK, S); 572*0b57cec5SDimitry Andric OS << ND << "\n"; 573*0b57cec5SDimitry Andric std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S); 574*0b57cec5SDimitry Andric for (uint32_t I = 0; I < ND; I++) { 575*0b57cec5SDimitry Andric if (VK == IPVK_IndirectCallTarget) 576*0b57cec5SDimitry Andric OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":" 577*0b57cec5SDimitry Andric << VD[I].Count << "\n"; 578*0b57cec5SDimitry Andric else 579*0b57cec5SDimitry Andric OS << VD[I].Value << ":" << VD[I].Count << "\n"; 580*0b57cec5SDimitry Andric } 581*0b57cec5SDimitry Andric } 582*0b57cec5SDimitry Andric } 583*0b57cec5SDimitry Andric 584*0b57cec5SDimitry Andric OS << "\n"; 585*0b57cec5SDimitry Andric } 586*0b57cec5SDimitry Andric 587*0b57cec5SDimitry Andric Error InstrProfWriter::writeText(raw_fd_ostream &OS) { 5881fd87a68SDimitry Andric // Check CS first since it implies an IR level profile. 58981ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) 590*0b57cec5SDimitry Andric OS << "# CSIR level Instrumentation Flag\n:csir\n"; 59181ad6265SDimitry Andric else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation)) 5921fd87a68SDimitry Andric OS << "# IR level Instrumentation Flag\n:ir\n"; 5931fd87a68SDimitry Andric 59481ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & 59581ad6265SDimitry Andric InstrProfKind::FunctionEntryInstrumentation)) 596e8d8bef9SDimitry Andric OS << "# Always instrument the function entry block\n:entry_first\n"; 597*0b57cec5SDimitry Andric InstrProfSymtab Symtab; 598*0b57cec5SDimitry Andric 599*0b57cec5SDimitry Andric using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>; 600*0b57cec5SDimitry Andric using RecordType = std::pair<StringRef, FuncPair>; 601*0b57cec5SDimitry Andric SmallVector<RecordType, 4> OrderedFuncData; 602*0b57cec5SDimitry Andric 603*0b57cec5SDimitry Andric for (const auto &I : FunctionData) { 604*0b57cec5SDimitry Andric if (shouldEncodeData(I.getValue())) { 605*0b57cec5SDimitry Andric if (Error E = Symtab.addFuncName(I.getKey())) 606*0b57cec5SDimitry Andric return E; 607*0b57cec5SDimitry Andric for (const auto &Func : I.getValue()) 608*0b57cec5SDimitry Andric OrderedFuncData.push_back(std::make_pair(I.getKey(), Func)); 609*0b57cec5SDimitry Andric } 610*0b57cec5SDimitry Andric } 611*0b57cec5SDimitry Andric 612*0b57cec5SDimitry Andric llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) { 613*0b57cec5SDimitry Andric return std::tie(A.first, A.second.first) < 614*0b57cec5SDimitry Andric std::tie(B.first, B.second.first); 615*0b57cec5SDimitry Andric }); 616*0b57cec5SDimitry Andric 617*0b57cec5SDimitry Andric for (const auto &record : OrderedFuncData) { 618*0b57cec5SDimitry Andric const StringRef &Name = record.first; 619*0b57cec5SDimitry Andric const FuncPair &Func = record.second; 620*0b57cec5SDimitry Andric writeRecordInText(Name, Func.first, Func.second, Symtab, OS); 621*0b57cec5SDimitry Andric } 622*0b57cec5SDimitry Andric 623fe6060f1SDimitry Andric for (const auto &record : OrderedFuncData) { 624fe6060f1SDimitry Andric const FuncPair &Func = record.second; 625fe6060f1SDimitry Andric if (Error E = validateRecord(Func.second)) 626fe6060f1SDimitry Andric return E; 627fe6060f1SDimitry Andric } 628fe6060f1SDimitry Andric 629*0b57cec5SDimitry Andric return Error::success(); 630*0b57cec5SDimitry Andric } 631