1 //=-- InstrProfReader.cpp - Instrumented profiling reader -------------------=//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains support for reading profiling data for clang's
11 // instrumentation based PGO and coverage.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/ProfileData/InstrProfReader.h"
16 #include "llvm/ProfileData/InstrProf.h"
17 
18 #include "InstrProfIndexed.h"
19 
20 #include <cassert>
21 
22 using namespace llvm;
23 
24 static ErrorOr<std::unique_ptr<MemoryBuffer>>
25 setupMemoryBuffer(std::string Path) {
26   ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
27       MemoryBuffer::getFileOrSTDIN(Path);
28   if (std::error_code EC = BufferOrErr.getError())
29     return EC;
30   auto Buffer = std::move(BufferOrErr.get());
31 
32   // Sanity check the file.
33   if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max())
34     return instrprof_error::too_large;
35   return std::move(Buffer);
36 }
37 
38 static std::error_code initializeReader(InstrProfReader &Reader) {
39   return Reader.readHeader();
40 }
41 
42 ErrorOr<std::unique_ptr<InstrProfReader>>
43 InstrProfReader::create(std::string Path) {
44   // Set up the buffer to read.
45   auto BufferOrError = setupMemoryBuffer(Path);
46   if (std::error_code EC = BufferOrError.getError())
47     return EC;
48 
49   auto Buffer = std::move(BufferOrError.get());
50   std::unique_ptr<InstrProfReader> Result;
51 
52   // Create the reader.
53   if (IndexedInstrProfReader::hasFormat(*Buffer))
54     Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
55   else if (RawInstrProfReader64::hasFormat(*Buffer))
56     Result.reset(new RawInstrProfReader64(std::move(Buffer)));
57   else if (RawInstrProfReader32::hasFormat(*Buffer))
58     Result.reset(new RawInstrProfReader32(std::move(Buffer)));
59   else
60     Result.reset(new TextInstrProfReader(std::move(Buffer)));
61 
62   // Initialize the reader and return the result.
63   if (std::error_code EC = initializeReader(*Result))
64     return EC;
65 
66   return std::move(Result);
67 }
68 
69 std::error_code IndexedInstrProfReader::create(
70     std::string Path, std::unique_ptr<IndexedInstrProfReader> &Result) {
71   // Set up the buffer to read.
72   auto BufferOrError = setupMemoryBuffer(Path);
73   if (std::error_code EC = BufferOrError.getError())
74     return EC;
75 
76   auto Buffer = std::move(BufferOrError.get());
77   // Create the reader.
78   if (!IndexedInstrProfReader::hasFormat(*Buffer))
79     return instrprof_error::bad_magic;
80   Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
81 
82   // Initialize the reader and return the result.
83   return initializeReader(*Result);
84 }
85 
86 void InstrProfIterator::Increment() {
87   if (Reader->readNextRecord(Record))
88     *this = InstrProfIterator();
89 }
90 
91 std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
92   // Skip empty lines and comments.
93   while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
94     ++Line;
95   // If we hit EOF while looking for a name, we're done.
96   if (Line.is_at_end())
97     return error(instrprof_error::eof);
98 
99   // Read the function name.
100   Record.Name = *Line++;
101 
102   // Read the function hash.
103   if (Line.is_at_end())
104     return error(instrprof_error::truncated);
105   if ((Line++)->getAsInteger(10, Record.Hash))
106     return error(instrprof_error::malformed);
107 
108   // Read the number of counters.
109   uint64_t NumCounters;
110   if (Line.is_at_end())
111     return error(instrprof_error::truncated);
112   if ((Line++)->getAsInteger(10, NumCounters))
113     return error(instrprof_error::malformed);
114   if (NumCounters == 0)
115     return error(instrprof_error::malformed);
116 
117   // Read each counter and fill our internal storage with the values.
118   Counts.clear();
119   Counts.reserve(NumCounters);
120   for (uint64_t I = 0; I < NumCounters; ++I) {
121     if (Line.is_at_end())
122       return error(instrprof_error::truncated);
123     uint64_t Count;
124     if ((Line++)->getAsInteger(10, Count))
125       return error(instrprof_error::malformed);
126     Counts.push_back(Count);
127   }
128   // Give the record a reference to our internal counter storage.
129   Record.Counts = Counts;
130 
131   return success();
132 }
133 
134 template <class IntPtrT>
135 static uint64_t getRawMagic();
136 
137 template <>
138 uint64_t getRawMagic<uint64_t>() {
139   return
140     uint64_t(255) << 56 |
141     uint64_t('l') << 48 |
142     uint64_t('p') << 40 |
143     uint64_t('r') << 32 |
144     uint64_t('o') << 24 |
145     uint64_t('f') << 16 |
146     uint64_t('r') <<  8 |
147     uint64_t(129);
148 }
149 
150 template <>
151 uint64_t getRawMagic<uint32_t>() {
152   return
153     uint64_t(255) << 56 |
154     uint64_t('l') << 48 |
155     uint64_t('p') << 40 |
156     uint64_t('r') << 32 |
157     uint64_t('o') << 24 |
158     uint64_t('f') << 16 |
159     uint64_t('R') <<  8 |
160     uint64_t(129);
161 }
162 
163 template <class IntPtrT>
164 bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) {
165   if (DataBuffer.getBufferSize() < sizeof(uint64_t))
166     return false;
167   uint64_t Magic =
168     *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
169   return getRawMagic<IntPtrT>() == Magic ||
170     sys::getSwappedBytes(getRawMagic<IntPtrT>()) == Magic;
171 }
172 
173 template <class IntPtrT>
174 std::error_code RawInstrProfReader<IntPtrT>::readHeader() {
175   if (!hasFormat(*DataBuffer))
176     return error(instrprof_error::bad_magic);
177   if (DataBuffer->getBufferSize() < sizeof(RawHeader))
178     return error(instrprof_error::bad_header);
179   auto *Header =
180     reinterpret_cast<const RawHeader *>(DataBuffer->getBufferStart());
181   ShouldSwapBytes = Header->Magic != getRawMagic<IntPtrT>();
182   return readHeader(*Header);
183 }
184 
185 template <class IntPtrT>
186 std::error_code
187 RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
188   const char *End = DataBuffer->getBufferEnd();
189   // Skip zero padding between profiles.
190   while (CurrentPos != End && *CurrentPos == 0)
191     ++CurrentPos;
192   // If there's nothing left, we're done.
193   if (CurrentPos == End)
194     return instrprof_error::eof;
195   // If there isn't enough space for another header, this is probably just
196   // garbage at the end of the file.
197   if (CurrentPos + sizeof(RawHeader) > End)
198     return instrprof_error::malformed;
199   // The writer ensures each profile is padded to start at an aligned address.
200   if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>())
201     return instrprof_error::malformed;
202   // The magic should have the same byte order as in the previous header.
203   uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
204   if (Magic != swap(getRawMagic<IntPtrT>()))
205     return instrprof_error::bad_magic;
206 
207   // There's another profile to read, so we need to process the header.
208   auto *Header = reinterpret_cast<const RawHeader *>(CurrentPos);
209   return readHeader(*Header);
210 }
211 
212 static uint64_t getRawVersion() {
213   return 1;
214 }
215 
216 template <class IntPtrT>
217 std::error_code
218 RawInstrProfReader<IntPtrT>::readHeader(const RawHeader &Header) {
219   if (swap(Header.Version) != getRawVersion())
220     return error(instrprof_error::unsupported_version);
221 
222   CountersDelta = swap(Header.CountersDelta);
223   NamesDelta = swap(Header.NamesDelta);
224   auto DataSize = swap(Header.DataSize);
225   auto CountersSize = swap(Header.CountersSize);
226   auto NamesSize = swap(Header.NamesSize);
227 
228   ptrdiff_t DataOffset = sizeof(RawHeader);
229   ptrdiff_t CountersOffset = DataOffset + sizeof(ProfileData) * DataSize;
230   ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize;
231   size_t ProfileSize = NamesOffset + sizeof(char) * NamesSize;
232 
233   auto *Start = reinterpret_cast<const char *>(&Header);
234   if (Start + ProfileSize > DataBuffer->getBufferEnd())
235     return error(instrprof_error::bad_header);
236 
237   Data = reinterpret_cast<const ProfileData *>(Start + DataOffset);
238   DataEnd = Data + DataSize;
239   CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
240   NamesStart = Start + NamesOffset;
241   ProfileEnd = Start + ProfileSize;
242 
243   return success();
244 }
245 
246 template <class IntPtrT>
247 std::error_code
248 RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
249   if (Data == DataEnd)
250     if (std::error_code EC = readNextHeader(ProfileEnd))
251       return EC;
252 
253   // Get the raw data.
254   StringRef RawName(getName(Data->NamePtr), swap(Data->NameSize));
255   uint32_t NumCounters = swap(Data->NumCounters);
256   if (NumCounters == 0)
257     return error(instrprof_error::malformed);
258   auto RawCounts = makeArrayRef(getCounter(Data->CounterPtr), NumCounters);
259 
260   // Check bounds.
261   auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
262   if (RawName.data() < NamesStart ||
263       RawName.data() + RawName.size() > DataBuffer->getBufferEnd() ||
264       RawCounts.data() < CountersStart ||
265       RawCounts.data() + RawCounts.size() > NamesStartAsCounter)
266     return error(instrprof_error::malformed);
267 
268   // Store the data in Record, byte-swapping as necessary.
269   Record.Hash = swap(Data->FuncHash);
270   Record.Name = RawName;
271   if (ShouldSwapBytes) {
272     Counts.clear();
273     Counts.reserve(RawCounts.size());
274     for (uint64_t Count : RawCounts)
275       Counts.push_back(swap(Count));
276     Record.Counts = Counts;
277   } else
278     Record.Counts = RawCounts;
279 
280   // Iterate.
281   ++Data;
282   return success();
283 }
284 
285 namespace llvm {
286 template class RawInstrProfReader<uint32_t>;
287 template class RawInstrProfReader<uint64_t>;
288 }
289 
290 InstrProfLookupTrait::hash_value_type
291 InstrProfLookupTrait::ComputeHash(StringRef K) {
292   return IndexedInstrProf::ComputeHash(HashType, K);
293 }
294 
295 bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
296   if (DataBuffer.getBufferSize() < 8)
297     return false;
298   using namespace support;
299   uint64_t Magic =
300       endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart());
301   return Magic == IndexedInstrProf::Magic;
302 }
303 
304 std::error_code IndexedInstrProfReader::readHeader() {
305   const unsigned char *Start =
306       (const unsigned char *)DataBuffer->getBufferStart();
307   const unsigned char *Cur = Start;
308   if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
309     return error(instrprof_error::truncated);
310 
311   using namespace support;
312 
313   // Check the magic number.
314   uint64_t Magic = endian::readNext<uint64_t, little, unaligned>(Cur);
315   if (Magic != IndexedInstrProf::Magic)
316     return error(instrprof_error::bad_magic);
317 
318   // Read the version.
319   FormatVersion = endian::readNext<uint64_t, little, unaligned>(Cur);
320   if (FormatVersion > IndexedInstrProf::Version)
321     return error(instrprof_error::unsupported_version);
322 
323   // Read the maximal function count.
324   MaxFunctionCount = endian::readNext<uint64_t, little, unaligned>(Cur);
325 
326   // Read the hash type and start offset.
327   IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
328       endian::readNext<uint64_t, little, unaligned>(Cur));
329   if (HashType > IndexedInstrProf::HashT::Last)
330     return error(instrprof_error::unsupported_hash_type);
331   uint64_t HashOffset = endian::readNext<uint64_t, little, unaligned>(Cur);
332 
333   // The rest of the file is an on disk hash table.
334   Index.reset(InstrProfReaderIndex::Create(Start + HashOffset, Cur, Start,
335                                            InstrProfLookupTrait(HashType)));
336   // Set up our iterator for readNextRecord.
337   RecordIterator = Index->data_begin();
338 
339   return success();
340 }
341 
342 std::error_code IndexedInstrProfReader::getFunctionCounts(
343     StringRef FuncName, uint64_t FuncHash, std::vector<uint64_t> &Counts) {
344   auto Iter = Index->find(FuncName);
345   if (Iter == Index->end())
346     return error(instrprof_error::unknown_function);
347 
348   // Found it. Look for counters with the right hash.
349   ArrayRef<uint64_t> Data = (*Iter).Data;
350   uint64_t NumCounts;
351   for (uint64_t I = 0, E = Data.size(); I != E; I += NumCounts) {
352     // The function hash comes first.
353     uint64_t FoundHash = Data[I++];
354     // In v1, we have at least one count. Later, we have the number of counts.
355     if (I == E)
356       return error(instrprof_error::malformed);
357     NumCounts = FormatVersion == 1 ? E - I : Data[I++];
358     // If we have more counts than data, this is bogus.
359     if (I + NumCounts > E)
360       return error(instrprof_error::malformed);
361     // Check for a match and fill the vector if there is one.
362     if (FoundHash == FuncHash) {
363       Counts = Data.slice(I, NumCounts);
364       return success();
365     }
366   }
367   return error(instrprof_error::hash_mismatch);
368 }
369 
370 std::error_code
371 IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) {
372   // Are we out of records?
373   if (RecordIterator == Index->data_end())
374     return error(instrprof_error::eof);
375 
376   // Record the current function name.
377   Record.Name = (*RecordIterator).Name;
378 
379   ArrayRef<uint64_t> Data = (*RecordIterator).Data;
380   // Valid data starts with a hash and either a count or the number of counts.
381   if (CurrentOffset + 1 > Data.size())
382     return error(instrprof_error::malformed);
383   // First we have a function hash.
384   Record.Hash = Data[CurrentOffset++];
385   // In version 1 we knew the number of counters implicitly, but in newer
386   // versions we store the number of counters next.
387   uint64_t NumCounts =
388       FormatVersion == 1 ? Data.size() - CurrentOffset : Data[CurrentOffset++];
389   if (CurrentOffset + NumCounts > Data.size())
390     return error(instrprof_error::malformed);
391   // And finally the counts themselves.
392   Record.Counts = Data.slice(CurrentOffset, NumCounts);
393 
394   // If we've exhausted this function's data, increment the record.
395   CurrentOffset += NumCounts;
396   if (CurrentOffset == Data.size()) {
397     ++RecordIterator;
398     CurrentOffset = 0;
399   }
400 
401   return success();
402 }
403