1e78d131aSEugene Zelenko //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2b9bd7f85SJustin Bogner //
32946cd70SChandler Carruth // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
42946cd70SChandler Carruth // See https://llvm.org/LICENSE.txt for license information.
52946cd70SChandler Carruth // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6b9bd7f85SJustin Bogner //
7b9bd7f85SJustin Bogner //===----------------------------------------------------------------------===//
8b9bd7f85SJustin Bogner //
9b9bd7f85SJustin Bogner // This file contains support for writing profiling data for clang's
10b9bd7f85SJustin Bogner // instrumentation based PGO and coverage.
11b9bd7f85SJustin Bogner //
12b9bd7f85SJustin Bogner //===----------------------------------------------------------------------===//
13b9bd7f85SJustin Bogner
146bda14b3SChandler Carruth #include "llvm/ProfileData/InstrProfWriter.h"
15e78d131aSEugene Zelenko #include "llvm/ADT/STLExtras.h"
16cdc71612SEugene Zelenko #include "llvm/ADT/StringRef.h"
17cdc71612SEugene Zelenko #include "llvm/IR/ProfileSummary.h"
18e78d131aSEugene Zelenko #include "llvm/ProfileData/InstrProf.h"
190a418490SSnehasish Kumar #include "llvm/ProfileData/MemProf.h"
20cdc71612SEugene Zelenko #include "llvm/ProfileData/ProfileCommon.h"
21e78d131aSEugene Zelenko #include "llvm/Support/Endian.h"
22b7aa2630SJustin Bogner #include "llvm/Support/EndianStream.h"
23e78d131aSEugene Zelenko #include "llvm/Support/Error.h"
24cdc71612SEugene Zelenko #include "llvm/Support/MemoryBuffer.h"
25b7aa2630SJustin Bogner #include "llvm/Support/OnDiskHashTable.h"
26cdc71612SEugene Zelenko #include "llvm/Support/raw_ostream.h"
27e78d131aSEugene Zelenko #include <cstdint>
28e78d131aSEugene Zelenko #include <memory>
29cdc71612SEugene Zelenko #include <string>
30437b1b3eSReid Kleckner #include <tuple>
31cdc71612SEugene Zelenko #include <utility>
32cdc71612SEugene Zelenko #include <vector>
33b7aa2630SJustin Bogner
34b9bd7f85SJustin Bogner using namespace llvm;
35b9bd7f85SJustin Bogner
36b6066385SXinliang David Li // A struct to define how the data stream should be patched. For Indexed
37b6066385SXinliang David Li // profiling, only uint64_t data type is needed.
38b6066385SXinliang David Li struct PatchItem {
39b6066385SXinliang David Li uint64_t Pos; // Where to patch.
40b6066385SXinliang David Li uint64_t *D; // Pointer to an array of source data.
41b6066385SXinliang David Li int N; // Number of elements in \c D array.
42b6066385SXinliang David Li };
43b6066385SXinliang David Li
44b6066385SXinliang David Li namespace llvm {
45cdc71612SEugene Zelenko
46b6066385SXinliang David Li // A wrapper class to abstract writer stream with support of bytes
47b6066385SXinliang David Li // back patching.
48285d7bd4SXinliang David Li class ProfOStream {
49285d7bd4SXinliang David Li public:
ProfOStream(raw_fd_ostream & FD)50e3f65297SPeter Collingbourne ProfOStream(raw_fd_ostream &FD)
51e3f65297SPeter Collingbourne : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
ProfOStream(raw_string_ostream & STR)52e78d131aSEugene Zelenko ProfOStream(raw_string_ostream &STR)
53e3f65297SPeter Collingbourne : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
54b6066385SXinliang David Li
tell()55b6066385SXinliang David Li uint64_t tell() { return OS.tell(); }
write(uint64_t V)56b6066385SXinliang David Li void write(uint64_t V) { LE.write<uint64_t>(V); }
57cdc71612SEugene Zelenko
58b6066385SXinliang David Li // \c patch can only be called when all data is written and flushed.
59b6066385SXinliang David Li // For raw_string_ostream, the patch is done on the target string
60b6066385SXinliang David Li // directly and it won't be reflected in the stream's internal buffer.
patch(PatchItem * P,int NItems)61b6066385SXinliang David Li void patch(PatchItem *P, int NItems) {
62b6066385SXinliang David Li using namespace support;
63e78d131aSEugene Zelenko
64b6066385SXinliang David Li if (IsFDOStream) {
65e78d131aSEugene Zelenko raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
660a418490SSnehasish Kumar const uint64_t LastPos = FDOStream.tell();
67b6066385SXinliang David Li for (int K = 0; K < NItems; K++) {
68b6066385SXinliang David Li FDOStream.seek(P[K].Pos);
69b6066385SXinliang David Li for (int I = 0; I < P[K].N; I++)
70b6066385SXinliang David Li write(P[K].D[I]);
71b6066385SXinliang David Li }
720a418490SSnehasish Kumar // Reset the stream to the last position after patching so that users
730a418490SSnehasish Kumar // don't accidentally overwrite data. This makes it consistent with
740a418490SSnehasish Kumar // the string stream below which replaces the data directly.
750a418490SSnehasish Kumar FDOStream.seek(LastPos);
76b6066385SXinliang David Li } else {
7772208a82SEugene Zelenko raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
78b6066385SXinliang David Li std::string &Data = SOStream.str(); // with flush
79b6066385SXinliang David Li for (int K = 0; K < NItems; K++) {
80b6066385SXinliang David Li for (int I = 0; I < P[K].N; I++) {
81b6066385SXinliang David Li uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
82b6066385SXinliang David Li Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
83b6066385SXinliang David Li (const char *)&Bytes, sizeof(uint64_t));
84b6066385SXinliang David Li }
85b6066385SXinliang David Li }
86b6066385SXinliang David Li }
87b6066385SXinliang David Li }
88cdc71612SEugene Zelenko
89b6066385SXinliang David Li // If \c OS is an instance of \c raw_fd_ostream, this field will be
90b6066385SXinliang David Li // true. Otherwise, \c OS will be an raw_string_ostream.
91b6066385SXinliang David Li bool IsFDOStream;
92b6066385SXinliang David Li raw_ostream &OS;
93e3f65297SPeter Collingbourne support::endian::Writer LE;
94b6066385SXinliang David Li };
95b6066385SXinliang David Li
96e1574f0cSXinliang David Li class InstrProfRecordWriterTrait {
97b7aa2630SJustin Bogner public:
9872208a82SEugene Zelenko using key_type = StringRef;
9972208a82SEugene Zelenko using key_type_ref = StringRef;
100b7aa2630SJustin Bogner
10172208a82SEugene Zelenko using data_type = const InstrProfWriter::ProfilingData *const;
10272208a82SEugene Zelenko using data_type_ref = const InstrProfWriter::ProfilingData *const;
103b7aa2630SJustin Bogner
10472208a82SEugene Zelenko using hash_value_type = uint64_t;
10572208a82SEugene Zelenko using offset_type = uint64_t;
106b7aa2630SJustin Bogner
107e78d131aSEugene Zelenko support::endianness ValueProfDataEndianness = support::little;
108e5a17e3fSEaswaran Raman InstrProfSummaryBuilder *SummaryBuilder;
109a6ff69f6SRong Xu InstrProfSummaryBuilder *CSSummaryBuilder;
110e1574f0cSXinliang David Li
111e78d131aSEugene Zelenko InstrProfRecordWriterTrait() = default;
112e78d131aSEugene Zelenko
ComputeHash(key_type_ref K)113b7aa2630SJustin Bogner static hash_value_type ComputeHash(key_type_ref K) {
11449ee76d0SXinliang David Li return IndexedInstrProf::ComputeHash(K);
115b7aa2630SJustin Bogner }
116b7aa2630SJustin Bogner
117b7aa2630SJustin Bogner static std::pair<offset_type, offset_type>
EmitKeyDataLength(raw_ostream & Out,key_type_ref K,data_type_ref V)118b7aa2630SJustin Bogner EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
119e78d131aSEugene Zelenko using namespace support;
120e78d131aSEugene Zelenko
121e3f65297SPeter Collingbourne endian::Writer LE(Out, little);
122b7aa2630SJustin Bogner
123e8081716SJustin Bogner offset_type N = K.size();
124b7aa2630SJustin Bogner LE.write<offset_type>(N);
125b7aa2630SJustin Bogner
126821d7471SJustin Bogner offset_type M = 0;
1279e9a057aSJustin Bogner for (const auto &ProfileData : *V) {
1282004f003SXinliang David Li const InstrProfRecord &ProfRecord = ProfileData.second;
1299e9a057aSJustin Bogner M += sizeof(uint64_t); // The function hash
1309e9a057aSJustin Bogner M += sizeof(uint64_t); // The size of the Counts vector
1312004f003SXinliang David Li M += ProfRecord.Counts.size() * sizeof(uint64_t);
1329e9a057aSJustin Bogner
1339e9a057aSJustin Bogner // Value data
13499556877SXinliang David Li M += ValueProfData::getSize(ProfileData.second);
1359e9a057aSJustin Bogner }
136b7aa2630SJustin Bogner LE.write<offset_type>(M);
137b7aa2630SJustin Bogner
138b7aa2630SJustin Bogner return std::make_pair(N, M);
139b7aa2630SJustin Bogner }
140b7aa2630SJustin Bogner
EmitKey(raw_ostream & Out,key_type_ref K,offset_type N)141e1574f0cSXinliang David Li void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
142b7aa2630SJustin Bogner Out.write(K.data(), N);
143b7aa2630SJustin Bogner }
144b7aa2630SJustin Bogner
EmitData(raw_ostream & Out,key_type_ref,data_type_ref V,offset_type)145e1574f0cSXinliang David Li void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
146e78d131aSEugene Zelenko using namespace support;
147e78d131aSEugene Zelenko
148e3f65297SPeter Collingbourne endian::Writer LE(Out, little);
1499e9a057aSJustin Bogner for (const auto &ProfileData : *V) {
1502004f003SXinliang David Li const InstrProfRecord &ProfRecord = ProfileData.second;
151a6ff69f6SRong Xu if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
152a6ff69f6SRong Xu CSSummaryBuilder->addRecord(ProfRecord);
153a6ff69f6SRong Xu else
154e5a17e3fSEaswaran Raman SummaryBuilder->addRecord(ProfRecord);
1552004f003SXinliang David Li
1569e9a057aSJustin Bogner LE.write<uint64_t>(ProfileData.first); // Function hash
1572004f003SXinliang David Li LE.write<uint64_t>(ProfRecord.Counts.size());
1586aa216c2SXinliang David Li for (uint64_t I : ProfRecord.Counts)
1596aa216c2SXinliang David Li LE.write<uint64_t>(I);
1609e9a057aSJustin Bogner
1619e9a057aSJustin Bogner // Write value data
16299556877SXinliang David Li std::unique_ptr<ValueProfData> VDataPtr =
16399556877SXinliang David Li ValueProfData::serializeFrom(ProfileData.second);
164ee415895SXinliang David Li uint32_t S = VDataPtr->getSize();
16546ad363bSXinliang David Li VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
166ee415895SXinliang David Li Out.write((const char *)VDataPtr.get(), S);
167b7aa2630SJustin Bogner }
168821d7471SJustin Bogner }
169b7aa2630SJustin Bogner };
170cdc71612SEugene Zelenko
171cdc71612SEugene Zelenko } // end namespace llvm
172b7aa2630SJustin Bogner
InstrProfWriter(bool Sparse)17313d89477SSnehasish Kumar InstrProfWriter::InstrProfWriter(bool Sparse)
17413d89477SSnehasish Kumar : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {}
175e1574f0cSXinliang David Li
~InstrProfWriter()176e1574f0cSXinliang David Li InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
177e1574f0cSXinliang David Li
17846ad363bSXinliang David Li // Internal interface for testing purpose only.
setValueProfDataEndianness(support::endianness Endianness)17946ad363bSXinliang David Li void InstrProfWriter::setValueProfDataEndianness(
18046ad363bSXinliang David Li support::endianness Endianness) {
181e1574f0cSXinliang David Li InfoObj->ValueProfDataEndianness = Endianness;
182ee415895SXinliang David Li }
183cdc71612SEugene Zelenko
setOutputSparse(bool Sparse)18400dab228SVedant Kumar void InstrProfWriter::setOutputSparse(bool Sparse) {
18500dab228SVedant Kumar this->Sparse = Sparse;
18600dab228SVedant Kumar }
187ee415895SXinliang David Li
addRecord(NamedInstrProfRecord && I,uint64_t Weight,function_ref<void (Error)> Warn)18898cce003SDavid Blaikie void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
18998cce003SDavid Blaikie function_ref<void(Error)> Warn) {
190cf9d52c6SDavid Blaikie auto Name = I.Name;
191cf9d52c6SDavid Blaikie auto Hash = I.Hash;
19298cce003SDavid Blaikie addRecord(Name, Hash, std::move(I), Weight, Warn);
193cf9d52c6SDavid Blaikie }
194cf9d52c6SDavid Blaikie
overlapRecord(NamedInstrProfRecord && Other,OverlapStats & Overlap,OverlapStats & FuncLevelOverlap,const OverlapFuncFilters & FuncFilter)195998b97f6SRong Xu void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
196998b97f6SRong Xu OverlapStats &Overlap,
197998b97f6SRong Xu OverlapStats &FuncLevelOverlap,
198998b97f6SRong Xu const OverlapFuncFilters &FuncFilter) {
199998b97f6SRong Xu auto Name = Other.Name;
200998b97f6SRong Xu auto Hash = Other.Hash;
201e0fa2689SRong Xu Other.accumulateCounts(FuncLevelOverlap.Test);
202998b97f6SRong Xu if (FunctionData.find(Name) == FunctionData.end()) {
203998b97f6SRong Xu Overlap.addOneUnique(FuncLevelOverlap.Test);
204998b97f6SRong Xu return;
205998b97f6SRong Xu }
206998b97f6SRong Xu if (FuncLevelOverlap.Test.CountSum < 1.0f) {
207998b97f6SRong Xu Overlap.Overlap.NumEntries += 1;
208998b97f6SRong Xu return;
209998b97f6SRong Xu }
210998b97f6SRong Xu auto &ProfileDataMap = FunctionData[Name];
211998b97f6SRong Xu bool NewFunc;
212998b97f6SRong Xu ProfilingData::iterator Where;
213998b97f6SRong Xu std::tie(Where, NewFunc) =
214998b97f6SRong Xu ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
215998b97f6SRong Xu if (NewFunc) {
216998b97f6SRong Xu Overlap.addOneMismatch(FuncLevelOverlap.Test);
217998b97f6SRong Xu return;
218998b97f6SRong Xu }
219998b97f6SRong Xu InstrProfRecord &Dest = Where->second;
220998b97f6SRong Xu
221998b97f6SRong Xu uint64_t ValueCutoff = FuncFilter.ValueCutoff;
222d14d7068SKazu Hirata if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
223998b97f6SRong Xu ValueCutoff = 0;
224998b97f6SRong Xu
225998b97f6SRong Xu Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
226998b97f6SRong Xu }
227998b97f6SRong Xu
addRecord(StringRef Name,uint64_t Hash,InstrProfRecord && I,uint64_t Weight,function_ref<void (Error)> Warn)22898cce003SDavid Blaikie void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
22998cce003SDavid Blaikie InstrProfRecord &&I, uint64_t Weight,
23098cce003SDavid Blaikie function_ref<void(Error)> Warn) {
231cf9d52c6SDavid Blaikie auto &ProfileDataMap = FunctionData[Name];
2329e9a057aSJustin Bogner
233a7318297SNathan Slingerland bool NewFunc;
234a7318297SNathan Slingerland ProfilingData::iterator Where;
235a7318297SNathan Slingerland std::tie(Where, NewFunc) =
236cf9d52c6SDavid Blaikie ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
237a7318297SNathan Slingerland InstrProfRecord &Dest = Where->second;
238aa5702d9SNathan Slingerland
23998cce003SDavid Blaikie auto MapWarn = [&](instrprof_error E) {
24098cce003SDavid Blaikie Warn(make_error<InstrProfError>(E));
24198cce003SDavid Blaikie };
24298cce003SDavid Blaikie
243a7318297SNathan Slingerland if (NewFunc) {
2449e9a057aSJustin Bogner // We've never seen a function with this name and hash, add it.
245a7318297SNathan Slingerland Dest = std::move(I);
24651dc04cfSXinliang David Li if (Weight > 1)
247a23f6234SWei Mi Dest.scale(Weight, 1, MapWarn);
248a7318297SNathan Slingerland } else {
249a7318297SNathan Slingerland // We're updating a function we've seen before.
25098cce003SDavid Blaikie Dest.merge(I, Weight, MapWarn);
251b9bd7f85SJustin Bogner }
252b9bd7f85SJustin Bogner
253062cde9cSXinliang David Li Dest.sortValueData();
254b9bd7f85SJustin Bogner }
255b9bd7f85SJustin Bogner
addMemProfRecord(const Function::GUID Id,const memprof::IndexedMemProfRecord & Record)256*6dd6a616SSnehasish Kumar void InstrProfWriter::addMemProfRecord(
257*6dd6a616SSnehasish Kumar const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
258*6dd6a616SSnehasish Kumar auto Result = MemProfRecordData.insert({Id, Record});
259*6dd6a616SSnehasish Kumar // If we inserted a new record then we are done.
260*6dd6a616SSnehasish Kumar if (Result.second) {
261*6dd6a616SSnehasish Kumar return;
262*6dd6a616SSnehasish Kumar }
263*6dd6a616SSnehasish Kumar memprof::IndexedMemProfRecord &Existing = Result.first->second;
26427a4f254SSnehasish Kumar Existing.merge(Record);
2650a418490SSnehasish Kumar }
266*6dd6a616SSnehasish Kumar
addMemProfFrame(const memprof::FrameId Id,const memprof::Frame & Frame,function_ref<void (Error)> Warn)267*6dd6a616SSnehasish Kumar bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
268*6dd6a616SSnehasish Kumar const memprof::Frame &Frame,
269*6dd6a616SSnehasish Kumar function_ref<void(Error)> Warn) {
270*6dd6a616SSnehasish Kumar auto Result = MemProfFrameData.insert({Id, Frame});
271*6dd6a616SSnehasish Kumar // If a mapping already exists for the current frame id and it does not
272*6dd6a616SSnehasish Kumar // match the new mapping provided then reset the existing contents and bail
273*6dd6a616SSnehasish Kumar // out. We don't support the merging of memprof data whose Frame -> Id
274*6dd6a616SSnehasish Kumar // mapping across profiles is inconsistent.
275*6dd6a616SSnehasish Kumar if (!Result.second && Result.first->second != Frame) {
276*6dd6a616SSnehasish Kumar Warn(make_error<InstrProfError>(instrprof_error::malformed,
277*6dd6a616SSnehasish Kumar "frame to id mapping mismatch"));
278*6dd6a616SSnehasish Kumar return false;
279*6dd6a616SSnehasish Kumar }
280*6dd6a616SSnehasish Kumar return true;
2810a418490SSnehasish Kumar }
2820a418490SSnehasish Kumar
mergeRecordsFromWriter(InstrProfWriter && IPW,function_ref<void (Error)> Warn)28398cce003SDavid Blaikie void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
28498cce003SDavid Blaikie function_ref<void(Error)> Warn) {
285e3a0bf50SVedant Kumar for (auto &I : IPW.FunctionData)
286e3a0bf50SVedant Kumar for (auto &Func : I.getValue())
28798cce003SDavid Blaikie addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
2880a418490SSnehasish Kumar
289*6dd6a616SSnehasish Kumar MemProfFrameData.reserve(IPW.MemProfFrameData.size());
290*6dd6a616SSnehasish Kumar for (auto &I : IPW.MemProfFrameData) {
291*6dd6a616SSnehasish Kumar // If we weren't able to add the frame mappings then it doesn't make sense
292*6dd6a616SSnehasish Kumar // to try to merge the records from this profile.
293*6dd6a616SSnehasish Kumar if (!addMemProfFrame(I.first, I.second, Warn))
294*6dd6a616SSnehasish Kumar return;
295*6dd6a616SSnehasish Kumar }
296*6dd6a616SSnehasish Kumar
297*6dd6a616SSnehasish Kumar MemProfRecordData.reserve(IPW.MemProfRecordData.size());
298*6dd6a616SSnehasish Kumar for (auto &I : IPW.MemProfRecordData) {
299*6dd6a616SSnehasish Kumar addMemProfRecord(I.first, I.second);
30027a4f254SSnehasish Kumar }
301e3a0bf50SVedant Kumar }
302e3a0bf50SVedant Kumar
shouldEncodeData(const ProfilingData & PD)30300dab228SVedant Kumar bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
30400dab228SVedant Kumar if (!Sparse)
30500dab228SVedant Kumar return true;
30600dab228SVedant Kumar for (const auto &Func : PD) {
30700dab228SVedant Kumar const InstrProfRecord &IPR = Func.second;
308e78d131aSEugene Zelenko if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
30900dab228SVedant Kumar return true;
31000dab228SVedant Kumar }
31100dab228SVedant Kumar return false;
31200dab228SVedant Kumar }
31300dab228SVedant Kumar
setSummary(IndexedInstrProf::Summary * TheSummary,ProfileSummary & PS)3146c93ee8dSXinliang David Li static void setSummary(IndexedInstrProf::Summary *TheSummary,
3157cefdb81SEaswaran Raman ProfileSummary &PS) {
3166c93ee8dSXinliang David Li using namespace IndexedInstrProf;
317e78d131aSEugene Zelenko
318d6790a0aSMircea Trofin const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
3196c93ee8dSXinliang David Li TheSummary->NumSummaryFields = Summary::NumKinds;
3206c93ee8dSXinliang David Li TheSummary->NumCutoffEntries = Res.size();
3216c93ee8dSXinliang David Li TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
3227cefdb81SEaswaran Raman TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
3237cefdb81SEaswaran Raman TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
3246c93ee8dSXinliang David Li TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
3257cefdb81SEaswaran Raman TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
3266c93ee8dSXinliang David Li TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
3276c93ee8dSXinliang David Li for (unsigned I = 0; I < Res.size(); I++)
3286c93ee8dSXinliang David Li TheSummary->setEntry(I, Res[I]);
3296c93ee8dSXinliang David Li }
3306c93ee8dSXinliang David Li
writeImpl(ProfOStream & OS)3316da7d314SMatthew Voss Error InstrProfWriter::writeImpl(ProfOStream &OS) {
332e78d131aSEugene Zelenko using namespace IndexedInstrProf;
333e78d131aSEugene Zelenko
334e1574f0cSXinliang David Li OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
3356c93ee8dSXinliang David Li
336e5a17e3fSEaswaran Raman InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
337e5a17e3fSEaswaran Raman InfoObj->SummaryBuilder = &ISB;
338a6ff69f6SRong Xu InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
339a6ff69f6SRong Xu InfoObj->CSSummaryBuilder = &CSISB;
3406c93ee8dSXinliang David Li
341b7aa2630SJustin Bogner // Populate the hash table generator.
342821d7471SJustin Bogner for (const auto &I : FunctionData)
34300dab228SVedant Kumar if (shouldEncodeData(I.getValue()))
344fa5b013dSJustin Bogner Generator.insert(I.getKey(), &I.getValue());
3450a418490SSnehasish Kumar
346b7aa2630SJustin Bogner // Write the header.
347dab183edSXinliang David Li IndexedInstrProf::Header Header;
348dab183edSXinliang David Li Header.Magic = IndexedInstrProf::Magic;
349a6b2c4f7SXinliang David Li Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
350b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
35133c76c0cSRong Xu Header.Version |= VARIANT_MASK_IR_PROF;
352b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
353a6ff69f6SRong Xu Header.Version |= VARIANT_MASK_CSIR_PROF;
354b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind &
355b6817999SSnehasish Kumar InstrProfKind::FunctionEntryInstrumentation))
35650da55a5SRong Xu Header.Version |= VARIANT_MASK_INSTR_ENTRY;
35711d30742SEllis Hoag if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
35811d30742SEllis Hoag Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
35911d30742SEllis Hoag if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
36011d30742SEllis Hoag Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
3610a418490SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
3620a418490SSnehasish Kumar Header.Version |= VARIANT_MASK_MEMPROF;
36350da55a5SRong Xu
3646c93ee8dSXinliang David Li Header.Unused = 0;
365dab183edSXinliang David Li Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
366dab183edSXinliang David Li Header.HashOffset = 0;
3670a418490SSnehasish Kumar Header.MemProfOffset = 0;
368dab183edSXinliang David Li int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
369dab183edSXinliang David Li
3700a418490SSnehasish Kumar // Only write out all the fields except 'HashOffset' and 'MemProfOffset'. We
3710a418490SSnehasish Kumar // need to remember the offset of these fields to allow back patching later.
3720a418490SSnehasish Kumar for (int I = 0; I < N - 2; I++)
373b6066385SXinliang David Li OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
374b7aa2630SJustin Bogner
3756c93ee8dSXinliang David Li // Save the location of Header.HashOffset field in \c OS.
3766c93ee8dSXinliang David Li uint64_t HashTableStartFieldOffset = OS.tell();
377dab183edSXinliang David Li // Reserve the space for HashOffset field.
378b6066385SXinliang David Li OS.write(0);
3796c93ee8dSXinliang David Li
3800a418490SSnehasish Kumar // Save the location of MemProf profile data. This is stored in two parts as
3810a418490SSnehasish Kumar // the schema and as a separate on-disk chained hashtable.
3820a418490SSnehasish Kumar uint64_t MemProfSectionOffset = OS.tell();
3830a418490SSnehasish Kumar // Reserve space for the MemProf table field to be patched later if this
3840a418490SSnehasish Kumar // profile contains memory profile information.
3850a418490SSnehasish Kumar OS.write(0);
3860a418490SSnehasish Kumar
3876c93ee8dSXinliang David Li // Reserve space to write profile summary data.
388e5a17e3fSEaswaran Raman uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
3896c93ee8dSXinliang David Li uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
3906c93ee8dSXinliang David Li // Remember the summary offset.
3916c93ee8dSXinliang David Li uint64_t SummaryOffset = OS.tell();
3926c93ee8dSXinliang David Li for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
3936c93ee8dSXinliang David Li OS.write(0);
394a6ff69f6SRong Xu uint64_t CSSummaryOffset = 0;
395a6ff69f6SRong Xu uint64_t CSSummarySize = 0;
396b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
397a6ff69f6SRong Xu CSSummaryOffset = OS.tell();
398a6ff69f6SRong Xu CSSummarySize = SummarySize / sizeof(uint64_t);
399a6ff69f6SRong Xu for (unsigned I = 0; I < CSSummarySize; I++)
400a6ff69f6SRong Xu OS.write(0);
401a6ff69f6SRong Xu }
4026c93ee8dSXinliang David Li
403b7aa2630SJustin Bogner // Write the hash table.
404e1574f0cSXinliang David Li uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
405b7aa2630SJustin Bogner
4060a418490SSnehasish Kumar // Write the MemProf profile data if we have it. This includes a simple schema
4070a418490SSnehasish Kumar // with the format described below followed by the hashtable:
408*6dd6a616SSnehasish Kumar // uint64_t RecordTableOffset = RecordTableGenerator.Emit
409*6dd6a616SSnehasish Kumar // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
410*6dd6a616SSnehasish Kumar // uint64_t FrameTableOffset = FrameTableGenerator.Emit
4110a418490SSnehasish Kumar // uint64_t Num schema entries
4120a418490SSnehasish Kumar // uint64_t Schema entry 0
4130a418490SSnehasish Kumar // uint64_t Schema entry 1
4140a418490SSnehasish Kumar // ....
4150a418490SSnehasish Kumar // uint64_t Schema entry N - 1
416*6dd6a616SSnehasish Kumar // OnDiskChainedHashTable MemProfRecordData
417*6dd6a616SSnehasish Kumar // OnDiskChainedHashTable MemProfFrameData
4180a418490SSnehasish Kumar uint64_t MemProfSectionStart = 0;
4190a418490SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
4200a418490SSnehasish Kumar MemProfSectionStart = OS.tell();
421*6dd6a616SSnehasish Kumar OS.write(0ULL); // Reserve space for the memprof record table offset.
422*6dd6a616SSnehasish Kumar OS.write(0ULL); // Reserve space for the memprof frame payload offset.
423*6dd6a616SSnehasish Kumar OS.write(0ULL); // Reserve space for the memprof frame table offset.
4240a418490SSnehasish Kumar
4250a418490SSnehasish Kumar auto Schema = memprof::PortableMemInfoBlock::getSchema();
4260a418490SSnehasish Kumar OS.write(static_cast<uint64_t>(Schema.size()));
4270a418490SSnehasish Kumar for (const auto Id : Schema) {
4280a418490SSnehasish Kumar OS.write(static_cast<uint64_t>(Id));
4290a418490SSnehasish Kumar }
4300a418490SSnehasish Kumar
431*6dd6a616SSnehasish Kumar auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
432*6dd6a616SSnehasish Kumar RecordWriter->Schema = &Schema;
433*6dd6a616SSnehasish Kumar OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
434*6dd6a616SSnehasish Kumar RecordTableGenerator;
435*6dd6a616SSnehasish Kumar for (auto &I : MemProfRecordData) {
43627a4f254SSnehasish Kumar // Insert the key (func hash) and value (memprof record).
437*6dd6a616SSnehasish Kumar RecordTableGenerator.insert(I.first, I.second);
4380a418490SSnehasish Kumar }
4390a418490SSnehasish Kumar
440*6dd6a616SSnehasish Kumar uint64_t RecordTableOffset =
441*6dd6a616SSnehasish Kumar RecordTableGenerator.Emit(OS.OS, *RecordWriter);
442*6dd6a616SSnehasish Kumar
443*6dd6a616SSnehasish Kumar uint64_t FramePayloadOffset = OS.tell();
444*6dd6a616SSnehasish Kumar
445*6dd6a616SSnehasish Kumar auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
446*6dd6a616SSnehasish Kumar OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
447*6dd6a616SSnehasish Kumar FrameTableGenerator;
448*6dd6a616SSnehasish Kumar for (auto &I : MemProfFrameData) {
449*6dd6a616SSnehasish Kumar // Insert the key (frame id) and value (frame contents).
450*6dd6a616SSnehasish Kumar FrameTableGenerator.insert(I.first, I.second);
451*6dd6a616SSnehasish Kumar }
452*6dd6a616SSnehasish Kumar
453*6dd6a616SSnehasish Kumar uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
454*6dd6a616SSnehasish Kumar
4550a418490SSnehasish Kumar PatchItem PatchItems[] = {
456*6dd6a616SSnehasish Kumar {MemProfSectionStart, &RecordTableOffset, 1},
457*6dd6a616SSnehasish Kumar {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
458*6dd6a616SSnehasish Kumar {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
4590a418490SSnehasish Kumar };
460*6dd6a616SSnehasish Kumar OS.patch(PatchItems, 3);
4610a418490SSnehasish Kumar }
4620a418490SSnehasish Kumar
4636c93ee8dSXinliang David Li // Allocate space for data to be serialized out.
4646c93ee8dSXinliang David Li std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
4656c93ee8dSXinliang David Li IndexedInstrProf::allocSummary(SummarySize);
4666c93ee8dSXinliang David Li // Compute the Summary and copy the data to the data
4676c93ee8dSXinliang David Li // structure to be serialized out (to disk or buffer).
46838de59e4SBenjamin Kramer std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
4697cefdb81SEaswaran Raman setSummary(TheSummary.get(), *PS);
470cdc71612SEugene Zelenko InfoObj->SummaryBuilder = nullptr;
4716c93ee8dSXinliang David Li
472a6ff69f6SRong Xu // For Context Sensitive summary.
473a6ff69f6SRong Xu std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
474b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
475a6ff69f6SRong Xu TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
476a6ff69f6SRong Xu std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
477a6ff69f6SRong Xu setSummary(TheCSSummary.get(), *CSPS);
478a6ff69f6SRong Xu }
479a6ff69f6SRong Xu InfoObj->CSSummaryBuilder = nullptr;
480a6ff69f6SRong Xu
481b6066385SXinliang David Li // Now do the final patch:
4826c93ee8dSXinliang David Li PatchItem PatchItems[] = {
4836c93ee8dSXinliang David Li // Patch the Header.HashOffset field.
4846c93ee8dSXinliang David Li {HashTableStartFieldOffset, &HashTableStart, 1},
4850a418490SSnehasish Kumar // Patch the Header.MemProfOffset (=0 for profiles without MemProf data).
4860a418490SSnehasish Kumar {MemProfSectionOffset, &MemProfSectionStart, 1},
4876c93ee8dSXinliang David Li // Patch the summary data.
4886c93ee8dSXinliang David Li {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
489a6ff69f6SRong Xu (int)(SummarySize / sizeof(uint64_t))},
490a6ff69f6SRong Xu {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
491a6ff69f6SRong Xu (int)CSSummarySize}};
492a6ff69f6SRong Xu
493b6066385SXinliang David Li OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
4946da7d314SMatthew Voss
4956da7d314SMatthew Voss for (const auto &I : FunctionData)
4966da7d314SMatthew Voss for (const auto &F : I.getValue())
4976da7d314SMatthew Voss if (Error E = validateRecord(F.second))
4986da7d314SMatthew Voss return E;
4996da7d314SMatthew Voss
5006da7d314SMatthew Voss return Error::success();
5012b6c537bSJustin Bogner }
5022b6c537bSJustin Bogner
write(raw_fd_ostream & OS)5036da7d314SMatthew Voss Error InstrProfWriter::write(raw_fd_ostream &OS) {
5042b6c537bSJustin Bogner // Write the hash table.
505b6066385SXinliang David Li ProfOStream POS(OS);
5066da7d314SMatthew Voss return writeImpl(POS);
507b6066385SXinliang David Li }
5082b6c537bSJustin Bogner
writeBuffer()509b6066385SXinliang David Li std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
510b6066385SXinliang David Li std::string Data;
511e78d131aSEugene Zelenko raw_string_ostream OS(Data);
512b6066385SXinliang David Li ProfOStream POS(OS);
513b6066385SXinliang David Li // Write the hash table.
5146da7d314SMatthew Voss if (Error E = writeImpl(POS))
5156da7d314SMatthew Voss return nullptr;
516b6066385SXinliang David Li // Return this in an aligned memory buffer.
517b6066385SXinliang David Li return MemoryBuffer::getMemBufferCopy(Data);
5182b6c537bSJustin Bogner }
5192b6c537bSJustin Bogner
520e3bf4fd3SXinliang David Li static const char *ValueProfKindStr[] = {
521312b5f86SDmitry Mikulin #define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
522e3bf4fd3SXinliang David Li #include "llvm/ProfileData/InstrProfData.inc"
523e3bf4fd3SXinliang David Li };
524e3bf4fd3SXinliang David Li
validateRecord(const InstrProfRecord & Func)5256da7d314SMatthew Voss Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
5266da7d314SMatthew Voss for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
5276da7d314SMatthew Voss uint32_t NS = Func.getNumValueSites(VK);
5286da7d314SMatthew Voss if (!NS)
5296da7d314SMatthew Voss continue;
5306da7d314SMatthew Voss for (uint32_t S = 0; S < NS; S++) {
5316da7d314SMatthew Voss uint32_t ND = Func.getNumValueDataForSite(VK, S);
5326da7d314SMatthew Voss std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
5336da7d314SMatthew Voss bool WasZero = false;
5346da7d314SMatthew Voss for (uint32_t I = 0; I < ND; I++)
5356da7d314SMatthew Voss if ((VK != IPVK_IndirectCallTarget) && (VD[I].Value == 0)) {
5366da7d314SMatthew Voss if (WasZero)
5376da7d314SMatthew Voss return make_error<InstrProfError>(instrprof_error::invalid_prof);
5386da7d314SMatthew Voss WasZero = true;
5396da7d314SMatthew Voss }
5406da7d314SMatthew Voss }
5416da7d314SMatthew Voss }
5426da7d314SMatthew Voss
5436da7d314SMatthew Voss return Error::success();
5446da7d314SMatthew Voss }
5456da7d314SMatthew Voss
writeRecordInText(StringRef Name,uint64_t Hash,const InstrProfRecord & Func,InstrProfSymtab & Symtab,raw_fd_ostream & OS)546cf9d52c6SDavid Blaikie void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
547cf9d52c6SDavid Blaikie const InstrProfRecord &Func,
548a716cc5cSXinliang David Li InstrProfSymtab &Symtab,
5496f7c19a4SXinliang David Li raw_fd_ostream &OS) {
550cf9d52c6SDavid Blaikie OS << Name << "\n";
551cf9d52c6SDavid Blaikie OS << "# Func Hash:\n" << Hash << "\n";
552c667683dSXinliang David Li OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
553c667683dSXinliang David Li OS << "# Counter Values:\n";
5546f7c19a4SXinliang David Li for (uint64_t Count : Func.Counts)
5556f7c19a4SXinliang David Li OS << Count << "\n";
5566f7c19a4SXinliang David Li
557e3bf4fd3SXinliang David Li uint32_t NumValueKinds = Func.getNumValueKinds();
558e3bf4fd3SXinliang David Li if (!NumValueKinds) {
559e3bf4fd3SXinliang David Li OS << "\n";
560e3bf4fd3SXinliang David Li return;
561e3bf4fd3SXinliang David Li }
562e3bf4fd3SXinliang David Li
563e3bf4fd3SXinliang David Li OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
564e3bf4fd3SXinliang David Li for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
565e3bf4fd3SXinliang David Li uint32_t NS = Func.getNumValueSites(VK);
566e3bf4fd3SXinliang David Li if (!NS)
567e3bf4fd3SXinliang David Li continue;
568e3bf4fd3SXinliang David Li OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
569e3bf4fd3SXinliang David Li OS << "# NumValueSites:\n" << NS << "\n";
570e3bf4fd3SXinliang David Li for (uint32_t S = 0; S < NS; S++) {
571e3bf4fd3SXinliang David Li uint32_t ND = Func.getNumValueDataForSite(VK, S);
572e3bf4fd3SXinliang David Li OS << ND << "\n";
573e3bf4fd3SXinliang David Li std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
574e3bf4fd3SXinliang David Li for (uint32_t I = 0; I < ND; I++) {
575e3bf4fd3SXinliang David Li if (VK == IPVK_IndirectCallTarget)
57629a21babSMircea Trofin OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
57729a21babSMircea Trofin << VD[I].Count << "\n";
578e3bf4fd3SXinliang David Li else
579e3bf4fd3SXinliang David Li OS << VD[I].Value << ":" << VD[I].Count << "\n";
580e3bf4fd3SXinliang David Li }
581e3bf4fd3SXinliang David Li }
582e3bf4fd3SXinliang David Li }
583e3bf4fd3SXinliang David Li
5846f7c19a4SXinliang David Li OS << "\n";
5856f7c19a4SXinliang David Li }
5866f7c19a4SXinliang David Li
writeText(raw_fd_ostream & OS)587b5794ca9SVedant Kumar Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
58813d89477SSnehasish Kumar // Check CS first since it implies an IR level profile.
589b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
590a6ff69f6SRong Xu OS << "# CSIR level Instrumentation Flag\n:csir\n";
591b6817999SSnehasish Kumar else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
59213d89477SSnehasish Kumar OS << "# IR level Instrumentation Flag\n:ir\n";
59313d89477SSnehasish Kumar
594b6817999SSnehasish Kumar if (static_cast<bool>(ProfileKind &
595b6817999SSnehasish Kumar InstrProfKind::FunctionEntryInstrumentation))
59650da55a5SRong Xu OS << "# Always instrument the function entry block\n:entry_first\n";
597a716cc5cSXinliang David Li InstrProfSymtab Symtab;
598a14f20c5SMandeep Singh Grang
599a14f20c5SMandeep Singh Grang using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
600a14f20c5SMandeep Singh Grang using RecordType = std::pair<StringRef, FuncPair>;
601a14f20c5SMandeep Singh Grang SmallVector<RecordType, 4> OrderedFuncData;
602a14f20c5SMandeep Singh Grang
603a14f20c5SMandeep Singh Grang for (const auto &I : FunctionData) {
604a14f20c5SMandeep Singh Grang if (shouldEncodeData(I.getValue())) {
605b5794ca9SVedant Kumar if (Error E = Symtab.addFuncName(I.getKey()))
606b5794ca9SVedant Kumar return E;
6076f7c19a4SXinliang David Li for (const auto &Func : I.getValue())
608a14f20c5SMandeep Singh Grang OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
609a14f20c5SMandeep Singh Grang }
610a14f20c5SMandeep Singh Grang }
611a14f20c5SMandeep Singh Grang
612a14f20c5SMandeep Singh Grang llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
613a14f20c5SMandeep Singh Grang return std::tie(A.first, A.second.first) <
614a14f20c5SMandeep Singh Grang std::tie(B.first, B.second.first);
615a14f20c5SMandeep Singh Grang });
616a14f20c5SMandeep Singh Grang
617a14f20c5SMandeep Singh Grang for (const auto &record : OrderedFuncData) {
618a14f20c5SMandeep Singh Grang const StringRef &Name = record.first;
619a14f20c5SMandeep Singh Grang const FuncPair &Func = record.second;
620a14f20c5SMandeep Singh Grang writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
621a14f20c5SMandeep Singh Grang }
622a14f20c5SMandeep Singh Grang
6236da7d314SMatthew Voss for (const auto &record : OrderedFuncData) {
6246da7d314SMatthew Voss const FuncPair &Func = record.second;
6256da7d314SMatthew Voss if (Error E = validateRecord(Func.second))
6266da7d314SMatthew Voss return E;
6276da7d314SMatthew Voss }
6286da7d314SMatthew Voss
629b5794ca9SVedant Kumar return Error::success();
6306f7c19a4SXinliang David Li }
631