1*0b57cec5SDimitry Andric //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2*0b57cec5SDimitry Andric //
3*0b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4*0b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5*0b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6*0b57cec5SDimitry Andric //
7*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
8*0b57cec5SDimitry Andric //
9*0b57cec5SDimitry Andric // This file contains support for writing profiling data for clang's
10*0b57cec5SDimitry Andric // instrumentation based PGO and coverage.
11*0b57cec5SDimitry Andric //
12*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
13*0b57cec5SDimitry Andric 
14*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfWriter.h"
15*0b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
16*0b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
17*0b57cec5SDimitry Andric #include "llvm/IR/ProfileSummary.h"
18*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProf.h"
1981ad6265SDimitry Andric #include "llvm/ProfileData/MemProf.h"
20*0b57cec5SDimitry Andric #include "llvm/ProfileData/ProfileCommon.h"
21*0b57cec5SDimitry Andric #include "llvm/Support/Endian.h"
22*0b57cec5SDimitry Andric #include "llvm/Support/EndianStream.h"
23*0b57cec5SDimitry Andric #include "llvm/Support/Error.h"
24*0b57cec5SDimitry Andric #include "llvm/Support/MemoryBuffer.h"
25*0b57cec5SDimitry Andric #include "llvm/Support/OnDiskHashTable.h"
26*0b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
27*0b57cec5SDimitry Andric #include <cstdint>
28*0b57cec5SDimitry Andric #include <memory>
29*0b57cec5SDimitry Andric #include <string>
30*0b57cec5SDimitry Andric #include <tuple>
31*0b57cec5SDimitry Andric #include <utility>
32*0b57cec5SDimitry Andric #include <vector>
33*0b57cec5SDimitry Andric 
34*0b57cec5SDimitry Andric using namespace llvm;
35*0b57cec5SDimitry Andric 
36*0b57cec5SDimitry Andric // A struct to define how the data stream should be patched. For Indexed
37*0b57cec5SDimitry Andric // profiling, only uint64_t data type is needed.
38*0b57cec5SDimitry Andric struct PatchItem {
39*0b57cec5SDimitry Andric   uint64_t Pos; // Where to patch.
40*0b57cec5SDimitry Andric   uint64_t *D;  // Pointer to an array of source data.
41*0b57cec5SDimitry Andric   int N;        // Number of elements in \c D array.
42*0b57cec5SDimitry Andric };
43*0b57cec5SDimitry Andric 
44*0b57cec5SDimitry Andric namespace llvm {
45*0b57cec5SDimitry Andric 
46*0b57cec5SDimitry Andric // A wrapper class to abstract writer stream with support of bytes
47*0b57cec5SDimitry Andric // back patching.
48*0b57cec5SDimitry Andric class ProfOStream {
49*0b57cec5SDimitry Andric public:
50*0b57cec5SDimitry Andric   ProfOStream(raw_fd_ostream &FD)
51*0b57cec5SDimitry Andric       : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
52*0b57cec5SDimitry Andric   ProfOStream(raw_string_ostream &STR)
53*0b57cec5SDimitry Andric       : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
54*0b57cec5SDimitry Andric 
55*0b57cec5SDimitry Andric   uint64_t tell() { return OS.tell(); }
56*0b57cec5SDimitry Andric   void write(uint64_t V) { LE.write<uint64_t>(V); }
57*0b57cec5SDimitry Andric 
58*0b57cec5SDimitry Andric   // \c patch can only be called when all data is written and flushed.
59*0b57cec5SDimitry Andric   // For raw_string_ostream, the patch is done on the target string
60*0b57cec5SDimitry Andric   // directly and it won't be reflected in the stream's internal buffer.
61*0b57cec5SDimitry Andric   void patch(PatchItem *P, int NItems) {
62*0b57cec5SDimitry Andric     using namespace support;
63*0b57cec5SDimitry Andric 
64*0b57cec5SDimitry Andric     if (IsFDOStream) {
65*0b57cec5SDimitry Andric       raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
6681ad6265SDimitry Andric       const uint64_t LastPos = FDOStream.tell();
67*0b57cec5SDimitry Andric       for (int K = 0; K < NItems; K++) {
68*0b57cec5SDimitry Andric         FDOStream.seek(P[K].Pos);
69*0b57cec5SDimitry Andric         for (int I = 0; I < P[K].N; I++)
70*0b57cec5SDimitry Andric           write(P[K].D[I]);
71*0b57cec5SDimitry Andric       }
7281ad6265SDimitry Andric       // Reset the stream to the last position after patching so that users
7381ad6265SDimitry Andric       // don't accidentally overwrite data. This makes it consistent with
7481ad6265SDimitry Andric       // the string stream below which replaces the data directly.
7581ad6265SDimitry Andric       FDOStream.seek(LastPos);
76*0b57cec5SDimitry Andric     } else {
77*0b57cec5SDimitry Andric       raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
78*0b57cec5SDimitry Andric       std::string &Data = SOStream.str(); // with flush
79*0b57cec5SDimitry Andric       for (int K = 0; K < NItems; K++) {
80*0b57cec5SDimitry Andric         for (int I = 0; I < P[K].N; I++) {
81*0b57cec5SDimitry Andric           uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
82*0b57cec5SDimitry Andric           Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
83*0b57cec5SDimitry Andric                        (const char *)&Bytes, sizeof(uint64_t));
84*0b57cec5SDimitry Andric         }
85*0b57cec5SDimitry Andric       }
86*0b57cec5SDimitry Andric     }
87*0b57cec5SDimitry Andric   }
88*0b57cec5SDimitry Andric 
89*0b57cec5SDimitry Andric   // If \c OS is an instance of \c raw_fd_ostream, this field will be
90*0b57cec5SDimitry Andric   // true. Otherwise, \c OS will be an raw_string_ostream.
91*0b57cec5SDimitry Andric   bool IsFDOStream;
92*0b57cec5SDimitry Andric   raw_ostream &OS;
93*0b57cec5SDimitry Andric   support::endian::Writer LE;
94*0b57cec5SDimitry Andric };
95*0b57cec5SDimitry Andric 
96*0b57cec5SDimitry Andric class InstrProfRecordWriterTrait {
97*0b57cec5SDimitry Andric public:
98*0b57cec5SDimitry Andric   using key_type = StringRef;
99*0b57cec5SDimitry Andric   using key_type_ref = StringRef;
100*0b57cec5SDimitry Andric 
101*0b57cec5SDimitry Andric   using data_type = const InstrProfWriter::ProfilingData *const;
102*0b57cec5SDimitry Andric   using data_type_ref = const InstrProfWriter::ProfilingData *const;
103*0b57cec5SDimitry Andric 
104*0b57cec5SDimitry Andric   using hash_value_type = uint64_t;
105*0b57cec5SDimitry Andric   using offset_type = uint64_t;
106*0b57cec5SDimitry Andric 
107*0b57cec5SDimitry Andric   support::endianness ValueProfDataEndianness = support::little;
108*0b57cec5SDimitry Andric   InstrProfSummaryBuilder *SummaryBuilder;
109*0b57cec5SDimitry Andric   InstrProfSummaryBuilder *CSSummaryBuilder;
110*0b57cec5SDimitry Andric 
111*0b57cec5SDimitry Andric   InstrProfRecordWriterTrait() = default;
112*0b57cec5SDimitry Andric 
113*0b57cec5SDimitry Andric   static hash_value_type ComputeHash(key_type_ref K) {
114*0b57cec5SDimitry Andric     return IndexedInstrProf::ComputeHash(K);
115*0b57cec5SDimitry Andric   }
116*0b57cec5SDimitry Andric 
117*0b57cec5SDimitry Andric   static std::pair<offset_type, offset_type>
118*0b57cec5SDimitry Andric   EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
119*0b57cec5SDimitry Andric     using namespace support;
120*0b57cec5SDimitry Andric 
121*0b57cec5SDimitry Andric     endian::Writer LE(Out, little);
122*0b57cec5SDimitry Andric 
123*0b57cec5SDimitry Andric     offset_type N = K.size();
124*0b57cec5SDimitry Andric     LE.write<offset_type>(N);
125*0b57cec5SDimitry Andric 
126*0b57cec5SDimitry Andric     offset_type M = 0;
127*0b57cec5SDimitry Andric     for (const auto &ProfileData : *V) {
128*0b57cec5SDimitry Andric       const InstrProfRecord &ProfRecord = ProfileData.second;
129*0b57cec5SDimitry Andric       M += sizeof(uint64_t); // The function hash
130*0b57cec5SDimitry Andric       M += sizeof(uint64_t); // The size of the Counts vector
131*0b57cec5SDimitry Andric       M += ProfRecord.Counts.size() * sizeof(uint64_t);
132*0b57cec5SDimitry Andric 
133*0b57cec5SDimitry Andric       // Value data
134*0b57cec5SDimitry Andric       M += ValueProfData::getSize(ProfileData.second);
135*0b57cec5SDimitry Andric     }
136*0b57cec5SDimitry Andric     LE.write<offset_type>(M);
137*0b57cec5SDimitry Andric 
138*0b57cec5SDimitry Andric     return std::make_pair(N, M);
139*0b57cec5SDimitry Andric   }
140*0b57cec5SDimitry Andric 
141*0b57cec5SDimitry Andric   void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
142*0b57cec5SDimitry Andric     Out.write(K.data(), N);
143*0b57cec5SDimitry Andric   }
144*0b57cec5SDimitry Andric 
145*0b57cec5SDimitry Andric   void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
146*0b57cec5SDimitry Andric     using namespace support;
147*0b57cec5SDimitry Andric 
148*0b57cec5SDimitry Andric     endian::Writer LE(Out, little);
149*0b57cec5SDimitry Andric     for (const auto &ProfileData : *V) {
150*0b57cec5SDimitry Andric       const InstrProfRecord &ProfRecord = ProfileData.second;
151*0b57cec5SDimitry Andric       if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
152*0b57cec5SDimitry Andric         CSSummaryBuilder->addRecord(ProfRecord);
153*0b57cec5SDimitry Andric       else
154*0b57cec5SDimitry Andric         SummaryBuilder->addRecord(ProfRecord);
155*0b57cec5SDimitry Andric 
156*0b57cec5SDimitry Andric       LE.write<uint64_t>(ProfileData.first); // Function hash
157*0b57cec5SDimitry Andric       LE.write<uint64_t>(ProfRecord.Counts.size());
158*0b57cec5SDimitry Andric       for (uint64_t I : ProfRecord.Counts)
159*0b57cec5SDimitry Andric         LE.write<uint64_t>(I);
160*0b57cec5SDimitry Andric 
161*0b57cec5SDimitry Andric       // Write value data
162*0b57cec5SDimitry Andric       std::unique_ptr<ValueProfData> VDataPtr =
163*0b57cec5SDimitry Andric           ValueProfData::serializeFrom(ProfileData.second);
164*0b57cec5SDimitry Andric       uint32_t S = VDataPtr->getSize();
165*0b57cec5SDimitry Andric       VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
166*0b57cec5SDimitry Andric       Out.write((const char *)VDataPtr.get(), S);
167*0b57cec5SDimitry Andric     }
168*0b57cec5SDimitry Andric   }
169*0b57cec5SDimitry Andric };
170*0b57cec5SDimitry Andric 
171*0b57cec5SDimitry Andric } // end namespace llvm
172*0b57cec5SDimitry Andric 
1731fd87a68SDimitry Andric InstrProfWriter::InstrProfWriter(bool Sparse)
1741fd87a68SDimitry Andric     : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {}
175*0b57cec5SDimitry Andric 
176*0b57cec5SDimitry Andric InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
177*0b57cec5SDimitry Andric 
178*0b57cec5SDimitry Andric // Internal interface for testing purpose only.
179*0b57cec5SDimitry Andric void InstrProfWriter::setValueProfDataEndianness(
180*0b57cec5SDimitry Andric     support::endianness Endianness) {
181*0b57cec5SDimitry Andric   InfoObj->ValueProfDataEndianness = Endianness;
182*0b57cec5SDimitry Andric }
183*0b57cec5SDimitry Andric 
184*0b57cec5SDimitry Andric void InstrProfWriter::setOutputSparse(bool Sparse) {
185*0b57cec5SDimitry Andric   this->Sparse = Sparse;
186*0b57cec5SDimitry Andric }
187*0b57cec5SDimitry Andric 
188*0b57cec5SDimitry Andric void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
189*0b57cec5SDimitry Andric                                 function_ref<void(Error)> Warn) {
190*0b57cec5SDimitry Andric   auto Name = I.Name;
191*0b57cec5SDimitry Andric   auto Hash = I.Hash;
192*0b57cec5SDimitry Andric   addRecord(Name, Hash, std::move(I), Weight, Warn);
193*0b57cec5SDimitry Andric }
194*0b57cec5SDimitry Andric 
195*0b57cec5SDimitry Andric void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
196*0b57cec5SDimitry Andric                                     OverlapStats &Overlap,
197*0b57cec5SDimitry Andric                                     OverlapStats &FuncLevelOverlap,
198*0b57cec5SDimitry Andric                                     const OverlapFuncFilters &FuncFilter) {
199*0b57cec5SDimitry Andric   auto Name = Other.Name;
200*0b57cec5SDimitry Andric   auto Hash = Other.Hash;
2018bcb0991SDimitry Andric   Other.accumulateCounts(FuncLevelOverlap.Test);
202*0b57cec5SDimitry Andric   if (FunctionData.find(Name) == FunctionData.end()) {
203*0b57cec5SDimitry Andric     Overlap.addOneUnique(FuncLevelOverlap.Test);
204*0b57cec5SDimitry Andric     return;
205*0b57cec5SDimitry Andric   }
206*0b57cec5SDimitry Andric   if (FuncLevelOverlap.Test.CountSum < 1.0f) {
207*0b57cec5SDimitry Andric     Overlap.Overlap.NumEntries += 1;
208*0b57cec5SDimitry Andric     return;
209*0b57cec5SDimitry Andric   }
210*0b57cec5SDimitry Andric   auto &ProfileDataMap = FunctionData[Name];
211*0b57cec5SDimitry Andric   bool NewFunc;
212*0b57cec5SDimitry Andric   ProfilingData::iterator Where;
213*0b57cec5SDimitry Andric   std::tie(Where, NewFunc) =
214*0b57cec5SDimitry Andric       ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
215*0b57cec5SDimitry Andric   if (NewFunc) {
216*0b57cec5SDimitry Andric     Overlap.addOneMismatch(FuncLevelOverlap.Test);
217*0b57cec5SDimitry Andric     return;
218*0b57cec5SDimitry Andric   }
219*0b57cec5SDimitry Andric   InstrProfRecord &Dest = Where->second;
220*0b57cec5SDimitry Andric 
221*0b57cec5SDimitry Andric   uint64_t ValueCutoff = FuncFilter.ValueCutoff;
222349cc55cSDimitry Andric   if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
223*0b57cec5SDimitry Andric     ValueCutoff = 0;
224*0b57cec5SDimitry Andric 
225*0b57cec5SDimitry Andric   Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
226*0b57cec5SDimitry Andric }
227*0b57cec5SDimitry Andric 
228*0b57cec5SDimitry Andric void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
229*0b57cec5SDimitry Andric                                 InstrProfRecord &&I, uint64_t Weight,
230*0b57cec5SDimitry Andric                                 function_ref<void(Error)> Warn) {
231*0b57cec5SDimitry Andric   auto &ProfileDataMap = FunctionData[Name];
232*0b57cec5SDimitry Andric 
233*0b57cec5SDimitry Andric   bool NewFunc;
234*0b57cec5SDimitry Andric   ProfilingData::iterator Where;
235*0b57cec5SDimitry Andric   std::tie(Where, NewFunc) =
236*0b57cec5SDimitry Andric       ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
237*0b57cec5SDimitry Andric   InstrProfRecord &Dest = Where->second;
238*0b57cec5SDimitry Andric 
239*0b57cec5SDimitry Andric   auto MapWarn = [&](instrprof_error E) {
240*0b57cec5SDimitry Andric     Warn(make_error<InstrProfError>(E));
241*0b57cec5SDimitry Andric   };
242*0b57cec5SDimitry Andric 
243*0b57cec5SDimitry Andric   if (NewFunc) {
244*0b57cec5SDimitry Andric     // We've never seen a function with this name and hash, add it.
245*0b57cec5SDimitry Andric     Dest = std::move(I);
246*0b57cec5SDimitry Andric     if (Weight > 1)
247e8d8bef9SDimitry Andric       Dest.scale(Weight, 1, MapWarn);
248*0b57cec5SDimitry Andric   } else {
249*0b57cec5SDimitry Andric     // We're updating a function we've seen before.
250*0b57cec5SDimitry Andric     Dest.merge(I, Weight, MapWarn);
251*0b57cec5SDimitry Andric   }
252*0b57cec5SDimitry Andric 
253*0b57cec5SDimitry Andric   Dest.sortValueData();
254*0b57cec5SDimitry Andric }
255*0b57cec5SDimitry Andric 
25681ad6265SDimitry Andric void InstrProfWriter::addMemProfRecord(
25781ad6265SDimitry Andric     const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
25881ad6265SDimitry Andric   auto Result = MemProfRecordData.insert({Id, Record});
25981ad6265SDimitry Andric   // If we inserted a new record then we are done.
26081ad6265SDimitry Andric   if (Result.second) {
26181ad6265SDimitry Andric     return;
26281ad6265SDimitry Andric   }
26381ad6265SDimitry Andric   memprof::IndexedMemProfRecord &Existing = Result.first->second;
26481ad6265SDimitry Andric   Existing.merge(Record);
26581ad6265SDimitry Andric }
26681ad6265SDimitry Andric 
26781ad6265SDimitry Andric bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
26881ad6265SDimitry Andric                                       const memprof::Frame &Frame,
26981ad6265SDimitry Andric                                       function_ref<void(Error)> Warn) {
27081ad6265SDimitry Andric   auto Result = MemProfFrameData.insert({Id, Frame});
27181ad6265SDimitry Andric   // If a mapping already exists for the current frame id and it does not
27281ad6265SDimitry Andric   // match the new mapping provided then reset the existing contents and bail
27381ad6265SDimitry Andric   // out. We don't support the merging of memprof data whose Frame -> Id
27481ad6265SDimitry Andric   // mapping across profiles is inconsistent.
27581ad6265SDimitry Andric   if (!Result.second && Result.first->second != Frame) {
27681ad6265SDimitry Andric     Warn(make_error<InstrProfError>(instrprof_error::malformed,
27781ad6265SDimitry Andric                                     "frame to id mapping mismatch"));
27881ad6265SDimitry Andric     return false;
27981ad6265SDimitry Andric   }
28081ad6265SDimitry Andric   return true;
28181ad6265SDimitry Andric }
28281ad6265SDimitry Andric 
283*0b57cec5SDimitry Andric void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
284*0b57cec5SDimitry Andric                                              function_ref<void(Error)> Warn) {
285*0b57cec5SDimitry Andric   for (auto &I : IPW.FunctionData)
286*0b57cec5SDimitry Andric     for (auto &Func : I.getValue())
287*0b57cec5SDimitry Andric       addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
28881ad6265SDimitry Andric 
28981ad6265SDimitry Andric   MemProfFrameData.reserve(IPW.MemProfFrameData.size());
29081ad6265SDimitry Andric   for (auto &I : IPW.MemProfFrameData) {
29181ad6265SDimitry Andric     // If we weren't able to add the frame mappings then it doesn't make sense
29281ad6265SDimitry Andric     // to try to merge the records from this profile.
29381ad6265SDimitry Andric     if (!addMemProfFrame(I.first, I.second, Warn))
29481ad6265SDimitry Andric       return;
29581ad6265SDimitry Andric   }
29681ad6265SDimitry Andric 
29781ad6265SDimitry Andric   MemProfRecordData.reserve(IPW.MemProfRecordData.size());
29881ad6265SDimitry Andric   for (auto &I : IPW.MemProfRecordData) {
29981ad6265SDimitry Andric     addMemProfRecord(I.first, I.second);
30081ad6265SDimitry Andric   }
301*0b57cec5SDimitry Andric }
302*0b57cec5SDimitry Andric 
303*0b57cec5SDimitry Andric bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
304*0b57cec5SDimitry Andric   if (!Sparse)
305*0b57cec5SDimitry Andric     return true;
306*0b57cec5SDimitry Andric   for (const auto &Func : PD) {
307*0b57cec5SDimitry Andric     const InstrProfRecord &IPR = Func.second;
308*0b57cec5SDimitry Andric     if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
309*0b57cec5SDimitry Andric       return true;
310*0b57cec5SDimitry Andric   }
311*0b57cec5SDimitry Andric   return false;
312*0b57cec5SDimitry Andric }
313*0b57cec5SDimitry Andric 
314*0b57cec5SDimitry Andric static void setSummary(IndexedInstrProf::Summary *TheSummary,
315*0b57cec5SDimitry Andric                        ProfileSummary &PS) {
316*0b57cec5SDimitry Andric   using namespace IndexedInstrProf;
317*0b57cec5SDimitry Andric 
318349cc55cSDimitry Andric   const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
319*0b57cec5SDimitry Andric   TheSummary->NumSummaryFields = Summary::NumKinds;
320*0b57cec5SDimitry Andric   TheSummary->NumCutoffEntries = Res.size();
321*0b57cec5SDimitry Andric   TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
322*0b57cec5SDimitry Andric   TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
323*0b57cec5SDimitry Andric   TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
324*0b57cec5SDimitry Andric   TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
325*0b57cec5SDimitry Andric   TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
326*0b57cec5SDimitry Andric   TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
327*0b57cec5SDimitry Andric   for (unsigned I = 0; I < Res.size(); I++)
328*0b57cec5SDimitry Andric     TheSummary->setEntry(I, Res[I]);
329*0b57cec5SDimitry Andric }
330*0b57cec5SDimitry Andric 
331fe6060f1SDimitry Andric Error InstrProfWriter::writeImpl(ProfOStream &OS) {
332*0b57cec5SDimitry Andric   using namespace IndexedInstrProf;
333*0b57cec5SDimitry Andric 
334*0b57cec5SDimitry Andric   OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
335*0b57cec5SDimitry Andric 
336*0b57cec5SDimitry Andric   InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
337*0b57cec5SDimitry Andric   InfoObj->SummaryBuilder = &ISB;
338*0b57cec5SDimitry Andric   InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
339*0b57cec5SDimitry Andric   InfoObj->CSSummaryBuilder = &CSISB;
340*0b57cec5SDimitry Andric 
341*0b57cec5SDimitry Andric   // Populate the hash table generator.
342*0b57cec5SDimitry Andric   for (const auto &I : FunctionData)
343*0b57cec5SDimitry Andric     if (shouldEncodeData(I.getValue()))
344*0b57cec5SDimitry Andric       Generator.insert(I.getKey(), &I.getValue());
34581ad6265SDimitry Andric 
346*0b57cec5SDimitry Andric   // Write the header.
347*0b57cec5SDimitry Andric   IndexedInstrProf::Header Header;
348*0b57cec5SDimitry Andric   Header.Magic = IndexedInstrProf::Magic;
349*0b57cec5SDimitry Andric   Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
35081ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
351*0b57cec5SDimitry Andric     Header.Version |= VARIANT_MASK_IR_PROF;
35281ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
353*0b57cec5SDimitry Andric     Header.Version |= VARIANT_MASK_CSIR_PROF;
35481ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind &
35581ad6265SDimitry Andric                         InstrProfKind::FunctionEntryInstrumentation))
356e8d8bef9SDimitry Andric     Header.Version |= VARIANT_MASK_INSTR_ENTRY;
3571fd87a68SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
3581fd87a68SDimitry Andric     Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
3591fd87a68SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
3601fd87a68SDimitry Andric     Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
36181ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
36281ad6265SDimitry Andric     Header.Version |= VARIANT_MASK_MEMPROF;
363e8d8bef9SDimitry Andric 
364*0b57cec5SDimitry Andric   Header.Unused = 0;
365*0b57cec5SDimitry Andric   Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
366*0b57cec5SDimitry Andric   Header.HashOffset = 0;
36781ad6265SDimitry Andric   Header.MemProfOffset = 0;
368*0b57cec5SDimitry Andric   int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
369*0b57cec5SDimitry Andric 
37081ad6265SDimitry Andric   // Only write out all the fields except 'HashOffset' and 'MemProfOffset'. We
37181ad6265SDimitry Andric   // need to remember the offset of these fields to allow back patching later.
37281ad6265SDimitry Andric   for (int I = 0; I < N - 2; I++)
373*0b57cec5SDimitry Andric     OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
374*0b57cec5SDimitry Andric 
375*0b57cec5SDimitry Andric   // Save the location of Header.HashOffset field in \c OS.
376*0b57cec5SDimitry Andric   uint64_t HashTableStartFieldOffset = OS.tell();
377*0b57cec5SDimitry Andric   // Reserve the space for HashOffset field.
378*0b57cec5SDimitry Andric   OS.write(0);
379*0b57cec5SDimitry Andric 
38081ad6265SDimitry Andric   // Save the location of MemProf profile data. This is stored in two parts as
38181ad6265SDimitry Andric   // the schema and as a separate on-disk chained hashtable.
38281ad6265SDimitry Andric   uint64_t MemProfSectionOffset = OS.tell();
38381ad6265SDimitry Andric   // Reserve space for the MemProf table field to be patched later if this
38481ad6265SDimitry Andric   // profile contains memory profile information.
38581ad6265SDimitry Andric   OS.write(0);
38681ad6265SDimitry Andric 
387*0b57cec5SDimitry Andric   // Reserve space to write profile summary data.
388*0b57cec5SDimitry Andric   uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
389*0b57cec5SDimitry Andric   uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
390*0b57cec5SDimitry Andric   // Remember the summary offset.
391*0b57cec5SDimitry Andric   uint64_t SummaryOffset = OS.tell();
392*0b57cec5SDimitry Andric   for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
393*0b57cec5SDimitry Andric     OS.write(0);
394*0b57cec5SDimitry Andric   uint64_t CSSummaryOffset = 0;
395*0b57cec5SDimitry Andric   uint64_t CSSummarySize = 0;
39681ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
397*0b57cec5SDimitry Andric     CSSummaryOffset = OS.tell();
398*0b57cec5SDimitry Andric     CSSummarySize = SummarySize / sizeof(uint64_t);
399*0b57cec5SDimitry Andric     for (unsigned I = 0; I < CSSummarySize; I++)
400*0b57cec5SDimitry Andric       OS.write(0);
401*0b57cec5SDimitry Andric   }
402*0b57cec5SDimitry Andric 
403*0b57cec5SDimitry Andric   // Write the hash table.
404*0b57cec5SDimitry Andric   uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
405*0b57cec5SDimitry Andric 
40681ad6265SDimitry Andric   // Write the MemProf profile data if we have it. This includes a simple schema
40781ad6265SDimitry Andric   // with the format described below followed by the hashtable:
40881ad6265SDimitry Andric   // uint64_t RecordTableOffset = RecordTableGenerator.Emit
40981ad6265SDimitry Andric   // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
41081ad6265SDimitry Andric   // uint64_t FrameTableOffset = FrameTableGenerator.Emit
41181ad6265SDimitry Andric   // uint64_t Num schema entries
41281ad6265SDimitry Andric   // uint64_t Schema entry 0
41381ad6265SDimitry Andric   // uint64_t Schema entry 1
41481ad6265SDimitry Andric   // ....
41581ad6265SDimitry Andric   // uint64_t Schema entry N - 1
41681ad6265SDimitry Andric   // OnDiskChainedHashTable MemProfRecordData
41781ad6265SDimitry Andric   // OnDiskChainedHashTable MemProfFrameData
41881ad6265SDimitry Andric   uint64_t MemProfSectionStart = 0;
41981ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
42081ad6265SDimitry Andric     MemProfSectionStart = OS.tell();
42181ad6265SDimitry Andric     OS.write(0ULL); // Reserve space for the memprof record table offset.
42281ad6265SDimitry Andric     OS.write(0ULL); // Reserve space for the memprof frame payload offset.
42381ad6265SDimitry Andric     OS.write(0ULL); // Reserve space for the memprof frame table offset.
42481ad6265SDimitry Andric 
42581ad6265SDimitry Andric     auto Schema = memprof::PortableMemInfoBlock::getSchema();
42681ad6265SDimitry Andric     OS.write(static_cast<uint64_t>(Schema.size()));
42781ad6265SDimitry Andric     for (const auto Id : Schema) {
42881ad6265SDimitry Andric       OS.write(static_cast<uint64_t>(Id));
42981ad6265SDimitry Andric     }
43081ad6265SDimitry Andric 
43181ad6265SDimitry Andric     auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
43281ad6265SDimitry Andric     RecordWriter->Schema = &Schema;
43381ad6265SDimitry Andric     OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
43481ad6265SDimitry Andric         RecordTableGenerator;
43581ad6265SDimitry Andric     for (auto &I : MemProfRecordData) {
43681ad6265SDimitry Andric       // Insert the key (func hash) and value (memprof record).
43781ad6265SDimitry Andric       RecordTableGenerator.insert(I.first, I.second);
43881ad6265SDimitry Andric     }
43981ad6265SDimitry Andric 
44081ad6265SDimitry Andric     uint64_t RecordTableOffset =
44181ad6265SDimitry Andric         RecordTableGenerator.Emit(OS.OS, *RecordWriter);
44281ad6265SDimitry Andric 
44381ad6265SDimitry Andric     uint64_t FramePayloadOffset = OS.tell();
44481ad6265SDimitry Andric 
44581ad6265SDimitry Andric     auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
44681ad6265SDimitry Andric     OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
44781ad6265SDimitry Andric         FrameTableGenerator;
44881ad6265SDimitry Andric     for (auto &I : MemProfFrameData) {
44981ad6265SDimitry Andric       // Insert the key (frame id) and value (frame contents).
45081ad6265SDimitry Andric       FrameTableGenerator.insert(I.first, I.second);
45181ad6265SDimitry Andric     }
45281ad6265SDimitry Andric 
45381ad6265SDimitry Andric     uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
45481ad6265SDimitry Andric 
45581ad6265SDimitry Andric     PatchItem PatchItems[] = {
45681ad6265SDimitry Andric         {MemProfSectionStart, &RecordTableOffset, 1},
45781ad6265SDimitry Andric         {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
45881ad6265SDimitry Andric         {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
45981ad6265SDimitry Andric     };
46081ad6265SDimitry Andric     OS.patch(PatchItems, 3);
46181ad6265SDimitry Andric   }
46281ad6265SDimitry Andric 
463*0b57cec5SDimitry Andric   // Allocate space for data to be serialized out.
464*0b57cec5SDimitry Andric   std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
465*0b57cec5SDimitry Andric       IndexedInstrProf::allocSummary(SummarySize);
466*0b57cec5SDimitry Andric   // Compute the Summary and copy the data to the data
467*0b57cec5SDimitry Andric   // structure to be serialized out (to disk or buffer).
468*0b57cec5SDimitry Andric   std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
469*0b57cec5SDimitry Andric   setSummary(TheSummary.get(), *PS);
470*0b57cec5SDimitry Andric   InfoObj->SummaryBuilder = nullptr;
471*0b57cec5SDimitry Andric 
472*0b57cec5SDimitry Andric   // For Context Sensitive summary.
473*0b57cec5SDimitry Andric   std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
47481ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
475*0b57cec5SDimitry Andric     TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
476*0b57cec5SDimitry Andric     std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
477*0b57cec5SDimitry Andric     setSummary(TheCSSummary.get(), *CSPS);
478*0b57cec5SDimitry Andric   }
479*0b57cec5SDimitry Andric   InfoObj->CSSummaryBuilder = nullptr;
480*0b57cec5SDimitry Andric 
481*0b57cec5SDimitry Andric   // Now do the final patch:
482*0b57cec5SDimitry Andric   PatchItem PatchItems[] = {
483*0b57cec5SDimitry Andric       // Patch the Header.HashOffset field.
484*0b57cec5SDimitry Andric       {HashTableStartFieldOffset, &HashTableStart, 1},
48581ad6265SDimitry Andric       // Patch the Header.MemProfOffset (=0 for profiles without MemProf data).
48681ad6265SDimitry Andric       {MemProfSectionOffset, &MemProfSectionStart, 1},
487*0b57cec5SDimitry Andric       // Patch the summary data.
488*0b57cec5SDimitry Andric       {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
489*0b57cec5SDimitry Andric        (int)(SummarySize / sizeof(uint64_t))},
490*0b57cec5SDimitry Andric       {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
491*0b57cec5SDimitry Andric        (int)CSSummarySize}};
492*0b57cec5SDimitry Andric 
493*0b57cec5SDimitry Andric   OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
494fe6060f1SDimitry Andric 
495fe6060f1SDimitry Andric   for (const auto &I : FunctionData)
496fe6060f1SDimitry Andric     for (const auto &F : I.getValue())
497fe6060f1SDimitry Andric       if (Error E = validateRecord(F.second))
498fe6060f1SDimitry Andric         return E;
499fe6060f1SDimitry Andric 
500fe6060f1SDimitry Andric   return Error::success();
501*0b57cec5SDimitry Andric }
502*0b57cec5SDimitry Andric 
503fe6060f1SDimitry Andric Error InstrProfWriter::write(raw_fd_ostream &OS) {
504*0b57cec5SDimitry Andric   // Write the hash table.
505*0b57cec5SDimitry Andric   ProfOStream POS(OS);
506fe6060f1SDimitry Andric   return writeImpl(POS);
507*0b57cec5SDimitry Andric }
508*0b57cec5SDimitry Andric 
509*0b57cec5SDimitry Andric std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
510*0b57cec5SDimitry Andric   std::string Data;
511*0b57cec5SDimitry Andric   raw_string_ostream OS(Data);
512*0b57cec5SDimitry Andric   ProfOStream POS(OS);
513*0b57cec5SDimitry Andric   // Write the hash table.
514fe6060f1SDimitry Andric   if (Error E = writeImpl(POS))
515fe6060f1SDimitry Andric     return nullptr;
516*0b57cec5SDimitry Andric   // Return this in an aligned memory buffer.
517*0b57cec5SDimitry Andric   return MemoryBuffer::getMemBufferCopy(Data);
518*0b57cec5SDimitry Andric }
519*0b57cec5SDimitry Andric 
520*0b57cec5SDimitry Andric static const char *ValueProfKindStr[] = {
521*0b57cec5SDimitry Andric #define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
522*0b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
523*0b57cec5SDimitry Andric };
524*0b57cec5SDimitry Andric 
525fe6060f1SDimitry Andric Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
526fe6060f1SDimitry Andric   for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
527fe6060f1SDimitry Andric     uint32_t NS = Func.getNumValueSites(VK);
528fe6060f1SDimitry Andric     if (!NS)
529fe6060f1SDimitry Andric       continue;
530fe6060f1SDimitry Andric     for (uint32_t S = 0; S < NS; S++) {
531fe6060f1SDimitry Andric       uint32_t ND = Func.getNumValueDataForSite(VK, S);
532fe6060f1SDimitry Andric       std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
533fe6060f1SDimitry Andric       bool WasZero = false;
534fe6060f1SDimitry Andric       for (uint32_t I = 0; I < ND; I++)
535fe6060f1SDimitry Andric         if ((VK != IPVK_IndirectCallTarget) && (VD[I].Value == 0)) {
536fe6060f1SDimitry Andric           if (WasZero)
537fe6060f1SDimitry Andric             return make_error<InstrProfError>(instrprof_error::invalid_prof);
538fe6060f1SDimitry Andric           WasZero = true;
539fe6060f1SDimitry Andric         }
540fe6060f1SDimitry Andric     }
541fe6060f1SDimitry Andric   }
542fe6060f1SDimitry Andric 
543fe6060f1SDimitry Andric   return Error::success();
544fe6060f1SDimitry Andric }
545fe6060f1SDimitry Andric 
546*0b57cec5SDimitry Andric void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
547*0b57cec5SDimitry Andric                                         const InstrProfRecord &Func,
548*0b57cec5SDimitry Andric                                         InstrProfSymtab &Symtab,
549*0b57cec5SDimitry Andric                                         raw_fd_ostream &OS) {
550*0b57cec5SDimitry Andric   OS << Name << "\n";
551*0b57cec5SDimitry Andric   OS << "# Func Hash:\n" << Hash << "\n";
552*0b57cec5SDimitry Andric   OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
553*0b57cec5SDimitry Andric   OS << "# Counter Values:\n";
554*0b57cec5SDimitry Andric   for (uint64_t Count : Func.Counts)
555*0b57cec5SDimitry Andric     OS << Count << "\n";
556*0b57cec5SDimitry Andric 
557*0b57cec5SDimitry Andric   uint32_t NumValueKinds = Func.getNumValueKinds();
558*0b57cec5SDimitry Andric   if (!NumValueKinds) {
559*0b57cec5SDimitry Andric     OS << "\n";
560*0b57cec5SDimitry Andric     return;
561*0b57cec5SDimitry Andric   }
562*0b57cec5SDimitry Andric 
563*0b57cec5SDimitry Andric   OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
564*0b57cec5SDimitry Andric   for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
565*0b57cec5SDimitry Andric     uint32_t NS = Func.getNumValueSites(VK);
566*0b57cec5SDimitry Andric     if (!NS)
567*0b57cec5SDimitry Andric       continue;
568*0b57cec5SDimitry Andric     OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
569*0b57cec5SDimitry Andric     OS << "# NumValueSites:\n" << NS << "\n";
570*0b57cec5SDimitry Andric     for (uint32_t S = 0; S < NS; S++) {
571*0b57cec5SDimitry Andric       uint32_t ND = Func.getNumValueDataForSite(VK, S);
572*0b57cec5SDimitry Andric       OS << ND << "\n";
573*0b57cec5SDimitry Andric       std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
574*0b57cec5SDimitry Andric       for (uint32_t I = 0; I < ND; I++) {
575*0b57cec5SDimitry Andric         if (VK == IPVK_IndirectCallTarget)
576*0b57cec5SDimitry Andric           OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
577*0b57cec5SDimitry Andric              << VD[I].Count << "\n";
578*0b57cec5SDimitry Andric         else
579*0b57cec5SDimitry Andric           OS << VD[I].Value << ":" << VD[I].Count << "\n";
580*0b57cec5SDimitry Andric       }
581*0b57cec5SDimitry Andric     }
582*0b57cec5SDimitry Andric   }
583*0b57cec5SDimitry Andric 
584*0b57cec5SDimitry Andric   OS << "\n";
585*0b57cec5SDimitry Andric }
586*0b57cec5SDimitry Andric 
587*0b57cec5SDimitry Andric Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
5881fd87a68SDimitry Andric   // Check CS first since it implies an IR level profile.
58981ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
590*0b57cec5SDimitry Andric     OS << "# CSIR level Instrumentation Flag\n:csir\n";
59181ad6265SDimitry Andric   else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
5921fd87a68SDimitry Andric     OS << "# IR level Instrumentation Flag\n:ir\n";
5931fd87a68SDimitry Andric 
59481ad6265SDimitry Andric   if (static_cast<bool>(ProfileKind &
59581ad6265SDimitry Andric                         InstrProfKind::FunctionEntryInstrumentation))
596e8d8bef9SDimitry Andric     OS << "# Always instrument the function entry block\n:entry_first\n";
597*0b57cec5SDimitry Andric   InstrProfSymtab Symtab;
598*0b57cec5SDimitry Andric 
599*0b57cec5SDimitry Andric   using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
600*0b57cec5SDimitry Andric   using RecordType = std::pair<StringRef, FuncPair>;
601*0b57cec5SDimitry Andric   SmallVector<RecordType, 4> OrderedFuncData;
602*0b57cec5SDimitry Andric 
603*0b57cec5SDimitry Andric   for (const auto &I : FunctionData) {
604*0b57cec5SDimitry Andric     if (shouldEncodeData(I.getValue())) {
605*0b57cec5SDimitry Andric       if (Error E = Symtab.addFuncName(I.getKey()))
606*0b57cec5SDimitry Andric         return E;
607*0b57cec5SDimitry Andric       for (const auto &Func : I.getValue())
608*0b57cec5SDimitry Andric         OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
609*0b57cec5SDimitry Andric     }
610*0b57cec5SDimitry Andric   }
611*0b57cec5SDimitry Andric 
612*0b57cec5SDimitry Andric   llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
613*0b57cec5SDimitry Andric     return std::tie(A.first, A.second.first) <
614*0b57cec5SDimitry Andric            std::tie(B.first, B.second.first);
615*0b57cec5SDimitry Andric   });
616*0b57cec5SDimitry Andric 
617*0b57cec5SDimitry Andric   for (const auto &record : OrderedFuncData) {
618*0b57cec5SDimitry Andric     const StringRef &Name = record.first;
619*0b57cec5SDimitry Andric     const FuncPair &Func = record.second;
620*0b57cec5SDimitry Andric     writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
621*0b57cec5SDimitry Andric   }
622*0b57cec5SDimitry Andric 
623fe6060f1SDimitry Andric   for (const auto &record : OrderedFuncData) {
624fe6060f1SDimitry Andric     const FuncPair &Func = record.second;
625fe6060f1SDimitry Andric     if (Error E = validateRecord(Func.second))
626fe6060f1SDimitry Andric       return E;
627fe6060f1SDimitry Andric   }
628fe6060f1SDimitry Andric 
629*0b57cec5SDimitry Andric   return Error::success();
630*0b57cec5SDimitry Andric }
631