1 //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/IR/Function.h"
10 #include "llvm/IR/IRBuilder.h"
11 #include "llvm/IR/LLVMContext.h"
12 #include "llvm/IR/Module.h"
13 #include "llvm/ProfileData/InstrProfReader.h"
14 #include "llvm/ProfileData/InstrProfWriter.h"
15 #include "llvm/ProfileData/MemProf.h"
16 #include "llvm/ProfileData/MemProfData.inc"
17 #include "llvm/Support/Compression.h"
18 #include "llvm/Support/raw_ostream.h"
19 #include "llvm/Testing/Support/Error.h"
20 #include "llvm/Testing/Support/SupportHelpers.h"
21 #include "gtest/gtest.h"
22 #include <cstdarg>
23 
24 using namespace llvm;
25 
26 LLVM_NODISCARD static ::testing::AssertionResult
ErrorEquals(instrprof_error Expected,Error E)27 ErrorEquals(instrprof_error Expected, Error E) {
28   instrprof_error Found;
29   std::string FoundMsg;
30   handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
31     Found = IPE.get();
32     FoundMsg = IPE.message();
33   });
34   if (Expected == Found)
35     return ::testing::AssertionSuccess();
36   return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";
37 }
38 
39 namespace {
40 
41 struct InstrProfTest : ::testing::Test {
42   InstrProfWriter Writer;
43   std::unique_ptr<IndexedInstrProfReader> Reader;
44 
SetUp__anon741d935d0211::InstrProfTest45   void SetUp() override { Writer.setOutputSparse(false); }
46 
readProfile__anon741d935d0211::InstrProfTest47   void readProfile(std::unique_ptr<MemoryBuffer> Profile,
48                    std::unique_ptr<MemoryBuffer> Remapping = nullptr) {
49     auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),
50                                                       std::move(Remapping));
51     EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
52     Reader = std::move(ReaderOrErr.get());
53   }
54 };
55 
56 struct SparseInstrProfTest : public InstrProfTest {
SetUp__anon741d935d0211::SparseInstrProfTest57   void SetUp() override { Writer.setOutputSparse(true); }
58 };
59 
60 struct MaybeSparseInstrProfTest : public InstrProfTest,
61                                   public ::testing::WithParamInterface<bool> {
SetUp__anon741d935d0211::MaybeSparseInstrProfTest62   void SetUp() override { Writer.setOutputSparse(GetParam()); }
63 };
64 
TEST_P(MaybeSparseInstrProfTest,write_and_read_empty_profile)65 TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
66   auto Profile = Writer.writeBuffer();
67   readProfile(std::move(Profile));
68   ASSERT_TRUE(Reader->begin() == Reader->end());
69 }
70 
__anon741d935d0302(Error E) 71 static const auto Err = [](Error E) {
72   consumeError(std::move(E));
73   FAIL();
74 };
75 
TEST_P(MaybeSparseInstrProfTest,write_and_read_one_function)76 TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
77   Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
78   auto Profile = Writer.writeBuffer();
79   readProfile(std::move(Profile));
80 
81   auto I = Reader->begin(), E = Reader->end();
82   ASSERT_TRUE(I != E);
83   ASSERT_EQ(StringRef("foo"), I->Name);
84   ASSERT_EQ(0x1234U, I->Hash);
85   ASSERT_EQ(4U, I->Counts.size());
86   ASSERT_EQ(1U, I->Counts[0]);
87   ASSERT_EQ(2U, I->Counts[1]);
88   ASSERT_EQ(3U, I->Counts[2]);
89   ASSERT_EQ(4U, I->Counts[3]);
90   ASSERT_TRUE(++I == E);
91 }
92 
TEST_P(MaybeSparseInstrProfTest,get_instr_prof_record)93 TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
94   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
95   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
96   auto Profile = Writer.writeBuffer();
97   readProfile(std::move(Profile));
98 
99   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("foo", 0x1234);
100   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
101   ASSERT_EQ(2U, R->Counts.size());
102   ASSERT_EQ(1U, R->Counts[0]);
103   ASSERT_EQ(2U, R->Counts[1]);
104 
105   R = Reader->getInstrProfRecord("foo", 0x1235);
106   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
107   ASSERT_EQ(2U, R->Counts.size());
108   ASSERT_EQ(3U, R->Counts[0]);
109   ASSERT_EQ(4U, R->Counts[1]);
110 
111   R = Reader->getInstrProfRecord("foo", 0x5678);
112   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));
113 
114   R = Reader->getInstrProfRecord("bar", 0x1234);
115   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));
116 }
117 
TEST_P(MaybeSparseInstrProfTest,get_function_counts)118 TEST_P(MaybeSparseInstrProfTest, get_function_counts) {
119   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
120   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
121   auto Profile = Writer.writeBuffer();
122   readProfile(std::move(Profile));
123 
124   std::vector<uint64_t> Counts;
125   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
126                     Succeeded());
127   ASSERT_EQ(2U, Counts.size());
128   ASSERT_EQ(1U, Counts[0]);
129   ASSERT_EQ(2U, Counts[1]);
130 
131   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
132                     Succeeded());
133   ASSERT_EQ(2U, Counts.size());
134   ASSERT_EQ(3U, Counts[0]);
135   ASSERT_EQ(4U, Counts[1]);
136 
137   Error E1 = Reader->getFunctionCounts("foo", 0x5678, Counts);
138   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));
139 
140   Error E2 = Reader->getFunctionCounts("bar", 0x1234, Counts);
141   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));
142 }
143 
144 // Profile data is copied from general.proftext
TEST_F(InstrProfTest,get_profile_summary)145 TEST_F(InstrProfTest, get_profile_summary) {
146   Writer.addRecord({"func1", 0x1234, {97531}}, Err);
147   Writer.addRecord({"func2", 0x1234, {0, 0}}, Err);
148   Writer.addRecord(
149       {"func3",
150        0x1234,
151        {2305843009213693952, 1152921504606846976, 576460752303423488,
152         288230376151711744, 144115188075855872, 72057594037927936}},
153       Err);
154   Writer.addRecord({"func4", 0x1234, {0}}, Err);
155   auto Profile = Writer.writeBuffer();
156   readProfile(std::move(Profile));
157 
158   auto VerifySummary = [](ProfileSummary &IPS) mutable {
159     ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());
160     ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());
161     ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());
162     ASSERT_EQ(10U, IPS.getNumCounts());
163     ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());
164     const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();
165     uint32_t Cutoff = 800000;
166     auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
167       return PE.Cutoff == Cutoff;
168     };
169     auto EightyPerc = find_if(Details, Predicate);
170     Cutoff = 900000;
171     auto NinetyPerc = find_if(Details, Predicate);
172     Cutoff = 950000;
173     auto NinetyFivePerc = find_if(Details, Predicate);
174     Cutoff = 990000;
175     auto NinetyNinePerc = find_if(Details, Predicate);
176     ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);
177     ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);
178     ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);
179     ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);
180   };
181   ProfileSummary &PS = Reader->getSummary(/* IsCS */ false);
182   VerifySummary(PS);
183 
184   // Test that conversion of summary to and from Metadata works.
185   LLVMContext Context;
186   Metadata *MD = PS.getMD(Context);
187   ASSERT_TRUE(MD);
188   ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);
189   ASSERT_TRUE(PSFromMD);
190   VerifySummary(*PSFromMD);
191   delete PSFromMD;
192 
193   // Test that summary can be attached to and read back from module.
194   Module M("my_module", Context);
195   M.setProfileSummary(MD, ProfileSummary::PSK_Instr);
196   MD = M.getProfileSummary(/* IsCS */ false);
197   ASSERT_TRUE(MD);
198   PSFromMD = ProfileSummary::getFromMD(MD);
199   ASSERT_TRUE(PSFromMD);
200   VerifySummary(*PSFromMD);
201   delete PSFromMD;
202 }
203 
TEST_F(InstrProfTest,test_writer_merge)204 TEST_F(InstrProfTest, test_writer_merge) {
205   Writer.addRecord({"func1", 0x1234, {42}}, Err);
206 
207   InstrProfWriter Writer2;
208   Writer2.addRecord({"func2", 0x1234, {0, 0}}, Err);
209 
210   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
211 
212   auto Profile = Writer.writeBuffer();
213   readProfile(std::move(Profile));
214 
215   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
216   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
217   ASSERT_EQ(1U, R->Counts.size());
218   ASSERT_EQ(42U, R->Counts[0]);
219 
220   R = Reader->getInstrProfRecord("func2", 0x1234);
221   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
222   ASSERT_EQ(2U, R->Counts.size());
223   ASSERT_EQ(0U, R->Counts[0]);
224   ASSERT_EQ(0U, R->Counts[1]);
225 }
226 
227 using ::llvm::memprof::IndexedMemProfRecord;
228 using ::llvm::memprof::MemInfoBlock;
229 using FrameIdMapTy =
230     llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
231 
getFrameMapping()232 static FrameIdMapTy getFrameMapping() {
233   FrameIdMapTy Mapping;
234   Mapping.insert({0, {0x123, 1, 2, false}});
235   Mapping.insert({1, {0x345, 3, 4, true}});
236   Mapping.insert({2, {0x125, 5, 6, false}});
237   Mapping.insert({3, {0x567, 7, 8, true}});
238   Mapping.insert({4, {0x124, 5, 6, false}});
239   Mapping.insert({5, {0x789, 8, 9, true}});
240   return Mapping;
241 }
242 
makeRecord(std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>> AllocFrames,std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>> CallSiteFrames,const MemInfoBlock & Block=MemInfoBlock ())243 IndexedMemProfRecord makeRecord(
244     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
245         AllocFrames,
246     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
247         CallSiteFrames,
248     const MemInfoBlock &Block = MemInfoBlock()) {
249   llvm::memprof::IndexedMemProfRecord MR;
250   for (const auto &Frames : AllocFrames)
251     MR.AllocSites.emplace_back(Frames, Block);
252   for (const auto &Frames : CallSiteFrames)
253     MR.CallSites.push_back(Frames);
254   return MR;
255 }
256 
257 MATCHER_P(EqualsRecord, Want, "") {
258   const memprof::MemProfRecord &Got = arg;
259 
__anon741d935d0602() 260   auto PrintAndFail = [&]() {
261     std::string Buffer;
262     llvm::raw_string_ostream OS(Buffer);
263     OS << "Want:\n";
264     Want.print(OS);
265     OS << "Got:\n";
266     Got.print(OS);
267     OS.flush();
268     *result_listener << "MemProf Record differs!\n" << Buffer;
269     return false;
270   };
271 
272   if (Want.AllocSites.size() != Got.AllocSites.size())
273     return PrintAndFail();
274   if (Want.CallSites.size() != Got.CallSites.size())
275     return PrintAndFail();
276 
277   for (size_t I = 0; I < Got.AllocSites.size(); I++) {
278     if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)
279       return PrintAndFail();
280     if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)
281       return PrintAndFail();
282   }
283 
284   for (size_t I = 0; I < Got.CallSites.size(); I++) {
285     if (Want.CallSites[I] != Got.CallSites[I])
286       return PrintAndFail();
287   }
288   return true;
289 }
290 
TEST_F(InstrProfTest,test_memprof)291 TEST_F(InstrProfTest, test_memprof) {
292   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
293                     Succeeded());
294 
295   const IndexedMemProfRecord IndexedMR = makeRecord(
296       /*AllocFrames=*/
297       {
298           {0, 1},
299           {2, 3},
300       },
301       /*CallSiteFrames=*/{
302           {4, 5},
303       });
304   const FrameIdMapTy IdToFrameMap = getFrameMapping();
305   for (const auto &I : IdToFrameMap) {
306     Writer.addMemProfFrame(I.first, I.getSecond(), Err);
307   }
308   Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
309 
310   auto Profile = Writer.writeBuffer();
311   readProfile(std::move(Profile));
312 
313   auto RecordOr = Reader->getMemProfRecord(0x9999);
314   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
315   const memprof::MemProfRecord &Record = RecordOr.get();
316 
317   memprof::FrameId LastUnmappedFrameId = 0;
318   bool HasFrameMappingError = false;
319   auto IdToFrameCallback = [&](const memprof::FrameId Id) {
320     auto Iter = IdToFrameMap.find(Id);
321     if (Iter == IdToFrameMap.end()) {
322       LastUnmappedFrameId = Id;
323       HasFrameMappingError = true;
324       return memprof::Frame(0, 0, 0, false);
325     }
326     return Iter->second;
327   };
328 
329   const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
330   ASSERT_FALSE(HasFrameMappingError)
331       << "could not map frame id: " << LastUnmappedFrameId;
332   EXPECT_THAT(WantRecord, EqualsRecord(Record));
333 }
334 
TEST_F(InstrProfTest,test_memprof_getrecord_error)335 TEST_F(InstrProfTest, test_memprof_getrecord_error) {
336   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
337                     Succeeded());
338 
339   const IndexedMemProfRecord IndexedMR = makeRecord(
340       /*AllocFrames=*/
341       {
342           {0, 1},
343           {2, 3},
344       },
345       /*CallSiteFrames=*/{
346           {4, 5},
347       });
348   // We skip adding the frame mappings here unlike the test_memprof unit test
349   // above to exercise the failure path when getMemProfRecord is invoked.
350   Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
351 
352   auto Profile = Writer.writeBuffer();
353   readProfile(std::move(Profile));
354 
355   // Missing frames give a hash_mismatch error.
356   auto RecordOr = Reader->getMemProfRecord(0x9999);
357   ASSERT_TRUE(
358       ErrorEquals(instrprof_error::hash_mismatch, RecordOr.takeError()));
359 
360   // Missing functions give a unknown_function error.
361   RecordOr = Reader->getMemProfRecord(0x1111);
362   ASSERT_TRUE(
363       ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError()));
364 }
365 
TEST_F(InstrProfTest,test_memprof_merge)366 TEST_F(InstrProfTest, test_memprof_merge) {
367   Writer.addRecord({"func1", 0x1234, {42}}, Err);
368 
369   InstrProfWriter Writer2;
370   ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),
371                     Succeeded());
372 
373   const IndexedMemProfRecord IndexedMR = makeRecord(
374       /*AllocFrames=*/
375       {
376           {0, 1},
377           {2, 3},
378       },
379       /*CallSiteFrames=*/{
380           {4, 5},
381       });
382 
383   const FrameIdMapTy IdToFrameMap = getFrameMapping();
384   for (const auto &I : IdToFrameMap) {
385     Writer.addMemProfFrame(I.first, I.getSecond(), Err);
386   }
387   Writer2.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
388 
389   ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),
390                     Succeeded());
391   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
392 
393   auto Profile = Writer.writeBuffer();
394   readProfile(std::move(Profile));
395 
396   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
397   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
398   ASSERT_EQ(1U, R->Counts.size());
399   ASSERT_EQ(42U, R->Counts[0]);
400 
401   auto RecordOr = Reader->getMemProfRecord(0x9999);
402   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
403   const memprof::MemProfRecord &Record = RecordOr.get();
404 
405   memprof::FrameId LastUnmappedFrameId = 0;
406   bool HasFrameMappingError = false;
407 
408   auto IdToFrameCallback = [&](const memprof::FrameId Id) {
409     auto Iter = IdToFrameMap.find(Id);
410     if (Iter == IdToFrameMap.end()) {
411       LastUnmappedFrameId = Id;
412       HasFrameMappingError = true;
413       return memprof::Frame(0, 0, 0, false);
414     }
415     return Iter->second;
416   };
417 
418   const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
419   ASSERT_FALSE(HasFrameMappingError)
420       << "could not map frame id: " << LastUnmappedFrameId;
421   EXPECT_THAT(WantRecord, EqualsRecord(Record));
422 }
423 
424 static const char callee1[] = "callee1";
425 static const char callee2[] = "callee2";
426 static const char callee3[] = "callee3";
427 static const char callee4[] = "callee4";
428 static const char callee5[] = "callee5";
429 static const char callee6[] = "callee6";
430 
TEST_P(MaybeSparseInstrProfTest,get_icall_data_read_write)431 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {
432   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
433 
434   // 4 value sites.
435   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
436   InstrProfValueData VD0[] = {
437       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
438   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
439   // No value profile data at the second site.
440   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
441   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
442   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
443   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
444   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
445 
446   Writer.addRecord(std::move(Record1), Err);
447   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
448   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
449   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
450   auto Profile = Writer.writeBuffer();
451   readProfile(std::move(Profile));
452 
453   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
454   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
455   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
456   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
457   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
458   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
459   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
460 
461   uint64_t TotalC;
462   std::unique_ptr<InstrProfValueData[]> VD =
463       R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
464 
465   ASSERT_EQ(3U, VD[0].Count);
466   ASSERT_EQ(2U, VD[1].Count);
467   ASSERT_EQ(1U, VD[2].Count);
468   ASSERT_EQ(6U, TotalC);
469 
470   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
471   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
472   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
473 }
474 
TEST_P(MaybeSparseInstrProfTest,annotate_vp_data)475 TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
476   NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
477   Record.reserveSites(IPVK_IndirectCallTarget, 1);
478   InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
479                               {4000, 4}, {6000, 6}};
480   Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 6, nullptr);
481   Writer.addRecord(std::move(Record), Err);
482   auto Profile = Writer.writeBuffer();
483   readProfile(std::move(Profile));
484   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
485   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
486 
487   LLVMContext Ctx;
488   std::unique_ptr<Module> M(new Module("MyModule", Ctx));
489   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
490                                         /*isVarArg=*/false);
491   Function *F =
492       Function::Create(FTy, Function::ExternalLinkage, "caller", M.get());
493   BasicBlock *BB = BasicBlock::Create(Ctx, "", F);
494 
495   IRBuilder<> Builder(BB);
496   BasicBlock *TBB = BasicBlock::Create(Ctx, "", F);
497   BasicBlock *FBB = BasicBlock::Create(Ctx, "", F);
498 
499   // Use branch instruction to annotate with value profile data for simplicity
500   Instruction *Inst = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
501   Instruction *Inst2 = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
502   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0);
503 
504   InstrProfValueData ValueData[5];
505   uint32_t N;
506   uint64_t T;
507   bool Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
508                                       ValueData, N, T);
509   ASSERT_TRUE(Res);
510   ASSERT_EQ(3U, N);
511   ASSERT_EQ(21U, T);
512   // The result should be sorted already:
513   ASSERT_EQ(6000U, ValueData[0].Value);
514   ASSERT_EQ(6U, ValueData[0].Count);
515   ASSERT_EQ(5000U, ValueData[1].Value);
516   ASSERT_EQ(5U, ValueData[1].Count);
517   ASSERT_EQ(4000U, ValueData[2].Value);
518   ASSERT_EQ(4U, ValueData[2].Count);
519   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 1, ValueData,
520                                  N, T);
521   ASSERT_TRUE(Res);
522   ASSERT_EQ(1U, N);
523   ASSERT_EQ(21U, T);
524 
525   Res = getValueProfDataFromInst(*Inst2, IPVK_IndirectCallTarget, 5, ValueData,
526                                  N, T);
527   ASSERT_FALSE(Res);
528 
529   // Remove the MD_prof metadata
530   Inst->setMetadata(LLVMContext::MD_prof, 0);
531   // Annotate 5 records this time.
532   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);
533   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
534                                       ValueData, N, T);
535   ASSERT_TRUE(Res);
536   ASSERT_EQ(5U, N);
537   ASSERT_EQ(21U, T);
538   ASSERT_EQ(6000U, ValueData[0].Value);
539   ASSERT_EQ(6U, ValueData[0].Count);
540   ASSERT_EQ(5000U, ValueData[1].Value);
541   ASSERT_EQ(5U, ValueData[1].Count);
542   ASSERT_EQ(4000U, ValueData[2].Value);
543   ASSERT_EQ(4U, ValueData[2].Count);
544   ASSERT_EQ(3000U, ValueData[3].Value);
545   ASSERT_EQ(3U, ValueData[3].Count);
546   ASSERT_EQ(2000U, ValueData[4].Value);
547   ASSERT_EQ(2U, ValueData[4].Count);
548 
549   // Remove the MD_prof metadata
550   Inst->setMetadata(LLVMContext::MD_prof, 0);
551   // Annotate with 4 records.
552   InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},
553                               {5000, 2}, {6000, 1}};
554   annotateValueSite(*M, *Inst, makeArrayRef(VD0Sorted).slice(2), 10,
555                     IPVK_IndirectCallTarget, 5);
556   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
557                                       ValueData, N, T);
558   ASSERT_TRUE(Res);
559   ASSERT_EQ(4U, N);
560   ASSERT_EQ(10U, T);
561   ASSERT_EQ(3000U, ValueData[0].Value);
562   ASSERT_EQ(4U, ValueData[0].Count);
563   ASSERT_EQ(4000U, ValueData[1].Value);
564   ASSERT_EQ(3U, ValueData[1].Count);
565   ASSERT_EQ(5000U, ValueData[2].Value);
566   ASSERT_EQ(2U, ValueData[2].Count);
567   ASSERT_EQ(6000U, ValueData[3].Value);
568   ASSERT_EQ(1U, ValueData[3].Count);
569 }
570 
TEST_P(MaybeSparseInstrProfTest,get_icall_data_read_write_with_weight)571 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) {
572   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
573 
574   // 4 value sites.
575   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
576   InstrProfValueData VD0[] = {
577       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
578   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
579   // No value profile data at the second site.
580   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
581   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
582   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
583   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
584   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
585 
586   Writer.addRecord(std::move(Record1), 10, Err);
587   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
588   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
589   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
590   auto Profile = Writer.writeBuffer();
591   readProfile(std::move(Profile));
592 
593   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
594   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
595   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
596   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
597   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
598   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
599   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
600 
601   uint64_t TotalC;
602   std::unique_ptr<InstrProfValueData[]> VD =
603       R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
604   ASSERT_EQ(30U, VD[0].Count);
605   ASSERT_EQ(20U, VD[1].Count);
606   ASSERT_EQ(10U, VD[2].Count);
607   ASSERT_EQ(60U, TotalC);
608 
609   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
610   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
611   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
612 }
613 
TEST_P(MaybeSparseInstrProfTest,get_icall_data_read_write_big_endian)614 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
615   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
616 
617   // 4 value sites.
618   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
619   InstrProfValueData VD0[] = {
620       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
621   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
622   // No value profile data at the second site.
623   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
624   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
625   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
626   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
627   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
628 
629   Writer.addRecord(std::move(Record1), Err);
630   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
631   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
632   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
633 
634   // Set big endian output.
635   Writer.setValueProfDataEndianness(support::big);
636 
637   auto Profile = Writer.writeBuffer();
638   readProfile(std::move(Profile));
639 
640   // Set big endian input.
641   Reader->setValueProfDataEndianness(support::big);
642 
643   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
644   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
645   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
646   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
647   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
648   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
649   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
650 
651   std::unique_ptr<InstrProfValueData[]> VD =
652       R->getValueForSite(IPVK_IndirectCallTarget, 0);
653   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
654   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
655   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
656 
657   // Restore little endian default:
658   Writer.setValueProfDataEndianness(support::little);
659 }
660 
TEST_P(MaybeSparseInstrProfTest,get_icall_data_merge1)661 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
662   static const char caller[] = "caller";
663   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
664   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
665 
666   // 5 value sites.
667   Record11.reserveSites(IPVK_IndirectCallTarget, 5);
668   InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
669                               {uint64_t(callee2), 2},
670                               {uint64_t(callee3), 3},
671                               {uint64_t(callee4), 4}};
672   Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
673 
674   // No value profile data at the second site.
675   Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
676 
677   InstrProfValueData VD2[] = {
678       {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
679   Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
680 
681   InstrProfValueData VD3[] = {{uint64_t(callee1), 1}};
682   Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
683 
684   InstrProfValueData VD4[] = {{uint64_t(callee1), 1},
685                               {uint64_t(callee2), 2},
686                               {uint64_t(callee3), 3}};
687   Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
688 
689   // A different record for the same caller.
690   Record12.reserveSites(IPVK_IndirectCallTarget, 5);
691   InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};
692   Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, 2, nullptr);
693 
694   // No value profile data at the second site.
695   Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
696 
697   InstrProfValueData VD22[] = {
698       {uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};
699   Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, 3, nullptr);
700 
701   Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr);
702 
703   InstrProfValueData VD42[] = {{uint64_t(callee1), 1},
704                                {uint64_t(callee2), 2},
705                                {uint64_t(callee3), 3}};
706   Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr);
707 
708   Writer.addRecord(std::move(Record11), Err);
709   // Merge profile data.
710   Writer.addRecord(std::move(Record12), Err);
711 
712   Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);
713   Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);
714   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
715   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
716   Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
717   auto Profile = Writer.writeBuffer();
718   readProfile(std::move(Profile));
719 
720   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
721   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
722   ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
723   ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
724   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
725   ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
726   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
727   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
728 
729   std::unique_ptr<InstrProfValueData[]> VD =
730       R->getValueForSite(IPVK_IndirectCallTarget, 0);
731   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
732   ASSERT_EQ(7U, VD[0].Count);
733   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
734   ASSERT_EQ(6U, VD[1].Count);
735   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
736   ASSERT_EQ(4U, VD[2].Count);
737   ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
738   ASSERT_EQ(1U, VD[3].Count);
739 
740   std::unique_ptr<InstrProfValueData[]> VD_2(
741       R->getValueForSite(IPVK_IndirectCallTarget, 2));
742   ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
743   ASSERT_EQ(6U, VD_2[0].Count);
744   ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
745   ASSERT_EQ(4U, VD_2[1].Count);
746   ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
747   ASSERT_EQ(3U, VD_2[2].Count);
748   ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
749   ASSERT_EQ(1U, VD_2[3].Count);
750 
751   std::unique_ptr<InstrProfValueData[]> VD_3(
752       R->getValueForSite(IPVK_IndirectCallTarget, 3));
753   ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1"));
754   ASSERT_EQ(1U, VD_3[0].Count);
755 
756   std::unique_ptr<InstrProfValueData[]> VD_4(
757       R->getValueForSite(IPVK_IndirectCallTarget, 4));
758   ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
759   ASSERT_EQ(6U, VD_4[0].Count);
760   ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
761   ASSERT_EQ(4U, VD_4[1].Count);
762   ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
763   ASSERT_EQ(2U, VD_4[2].Count);
764 }
765 
TEST_P(MaybeSparseInstrProfTest,get_icall_data_merge1_saturation)766 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
767   static const char bar[] = "bar";
768 
769   const uint64_t Max = std::numeric_limits<uint64_t>::max();
770 
771   instrprof_error Result;
772   auto Err = [&](Error E) { Result = InstrProfError::take(std::move(E)); };
773   Result = instrprof_error::success;
774   Writer.addRecord({"foo", 0x1234, {1}}, Err);
775   ASSERT_EQ(Result, instrprof_error::success);
776 
777   // Verify counter overflow.
778   Result = instrprof_error::success;
779   Writer.addRecord({"foo", 0x1234, {Max}}, Err);
780   ASSERT_EQ(Result, instrprof_error::counter_overflow);
781 
782   Result = instrprof_error::success;
783   Writer.addRecord({bar, 0x9012, {8}}, Err);
784   ASSERT_EQ(Result, instrprof_error::success);
785 
786   NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
787   Record4.reserveSites(IPVK_IndirectCallTarget, 1);
788   InstrProfValueData VD4[] = {{uint64_t(bar), 1}};
789   Record4.addValueData(IPVK_IndirectCallTarget, 0, VD4, 1, nullptr);
790   Result = instrprof_error::success;
791   Writer.addRecord(std::move(Record4), Err);
792   ASSERT_EQ(Result, instrprof_error::success);
793 
794   // Verify value data counter overflow.
795   NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
796   Record5.reserveSites(IPVK_IndirectCallTarget, 1);
797   InstrProfValueData VD5[] = {{uint64_t(bar), Max}};
798   Record5.addValueData(IPVK_IndirectCallTarget, 0, VD5, 1, nullptr);
799   Result = instrprof_error::success;
800   Writer.addRecord(std::move(Record5), Err);
801   ASSERT_EQ(Result, instrprof_error::counter_overflow);
802 
803   auto Profile = Writer.writeBuffer();
804   readProfile(std::move(Profile));
805 
806   // Verify saturation of counts.
807   Expected<InstrProfRecord> ReadRecord1 =
808       Reader->getInstrProfRecord("foo", 0x1234);
809   EXPECT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
810   ASSERT_EQ(Max, ReadRecord1->Counts[0]);
811 
812   Expected<InstrProfRecord> ReadRecord2 =
813       Reader->getInstrProfRecord("baz", 0x5678);
814   ASSERT_TRUE(bool(ReadRecord2));
815   ASSERT_EQ(1U, ReadRecord2->getNumValueSites(IPVK_IndirectCallTarget));
816   std::unique_ptr<InstrProfValueData[]> VD =
817       ReadRecord2->getValueForSite(IPVK_IndirectCallTarget, 0);
818   ASSERT_EQ(StringRef("bar"), StringRef((const char *)VD[0].Value, 3));
819   ASSERT_EQ(Max, VD[0].Count);
820 }
821 
822 // This test tests that when there are too many values
823 // for a given site, the merged results are properly
824 // truncated.
TEST_P(MaybeSparseInstrProfTest,get_icall_data_merge_site_trunc)825 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
826   static const char caller[] = "caller";
827 
828   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
829   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
830 
831   // 2 value sites.
832   Record11.reserveSites(IPVK_IndirectCallTarget, 2);
833   InstrProfValueData VD0[255];
834   for (int I = 0; I < 255; I++) {
835     VD0[I].Value = 2 * I;
836     VD0[I].Count = 2 * I + 1000;
837   }
838 
839   Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 255, nullptr);
840   Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
841 
842   Record12.reserveSites(IPVK_IndirectCallTarget, 2);
843   InstrProfValueData VD1[255];
844   for (int I = 0; I < 255; I++) {
845     VD1[I].Value = 2 * I + 1;
846     VD1[I].Count = 2 * I + 1001;
847   }
848 
849   Record12.addValueData(IPVK_IndirectCallTarget, 0, VD1, 255, nullptr);
850   Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
851 
852   Writer.addRecord(std::move(Record11), Err);
853   // Merge profile data.
854   Writer.addRecord(std::move(Record12), Err);
855 
856   auto Profile = Writer.writeBuffer();
857   readProfile(std::move(Profile));
858 
859   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
860   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
861   std::unique_ptr<InstrProfValueData[]> VD(
862       R->getValueForSite(IPVK_IndirectCallTarget, 0));
863   ASSERT_EQ(2U, R->getNumValueSites(IPVK_IndirectCallTarget));
864   ASSERT_EQ(255U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
865   for (unsigned I = 0; I < 255; I++) {
866     ASSERT_EQ(VD[I].Value, 509 - I);
867     ASSERT_EQ(VD[I].Count, 1509 - I);
868   }
869 }
870 
addValueProfData(InstrProfRecord & Record)871 static void addValueProfData(InstrProfRecord &Record) {
872   Record.reserveSites(IPVK_IndirectCallTarget, 5);
873   InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
874                               {uint64_t(callee2), 1000},
875                               {uint64_t(callee3), 500},
876                               {uint64_t(callee4), 300},
877                               {uint64_t(callee5), 100}};
878   Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
879   InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
880                               {uint64_t(callee3), 1000},
881                               {uint64_t(callee2), 2500},
882                               {uint64_t(callee1), 1300}};
883   Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
884   InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
885                               {uint64_t(callee3), 1000},
886                               {uint64_t(callee4), 5500}};
887   Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
888   InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
889                               {uint64_t(callee3), 2000}};
890   Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
891   Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
892 }
893 
TEST_P(MaybeSparseInstrProfTest,value_prof_data_read_write)894 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) {
895   InstrProfRecord SrcRecord({1ULL << 31, 2});
896   addValueProfData(SrcRecord);
897   std::unique_ptr<ValueProfData> VPData =
898       ValueProfData::serializeFrom(SrcRecord);
899 
900   InstrProfRecord Record({1ULL << 31, 2});
901   VPData->deserializeTo(Record, nullptr);
902 
903   // Now read data from Record and sanity check the data
904   ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
905   ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
906   ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
907   ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
908   ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
909   ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
910 
911   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
912     return VD1.Count > VD2.Count;
913   };
914   std::unique_ptr<InstrProfValueData[]> VD_0(
915       Record.getValueForSite(IPVK_IndirectCallTarget, 0));
916   llvm::sort(&VD_0[0], &VD_0[5], Cmp);
917   ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
918   ASSERT_EQ(1000U, VD_0[0].Count);
919   ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
920   ASSERT_EQ(500U, VD_0[1].Count);
921   ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
922   ASSERT_EQ(400U, VD_0[2].Count);
923   ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
924   ASSERT_EQ(300U, VD_0[3].Count);
925   ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
926   ASSERT_EQ(100U, VD_0[4].Count);
927 
928   std::unique_ptr<InstrProfValueData[]> VD_1(
929       Record.getValueForSite(IPVK_IndirectCallTarget, 1));
930   llvm::sort(&VD_1[0], &VD_1[4], Cmp);
931   ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
932   ASSERT_EQ(2500U, VD_1[0].Count);
933   ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
934   ASSERT_EQ(1300U, VD_1[1].Count);
935   ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
936   ASSERT_EQ(1000U, VD_1[2].Count);
937   ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
938   ASSERT_EQ(800U, VD_1[3].Count);
939 
940   std::unique_ptr<InstrProfValueData[]> VD_2(
941       Record.getValueForSite(IPVK_IndirectCallTarget, 2));
942   llvm::sort(&VD_2[0], &VD_2[3], Cmp);
943   ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
944   ASSERT_EQ(5500U, VD_2[0].Count);
945   ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
946   ASSERT_EQ(1000U, VD_2[1].Count);
947   ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
948   ASSERT_EQ(800U, VD_2[2].Count);
949 
950   std::unique_ptr<InstrProfValueData[]> VD_3(
951       Record.getValueForSite(IPVK_IndirectCallTarget, 3));
952   llvm::sort(&VD_3[0], &VD_3[2], Cmp);
953   ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
954   ASSERT_EQ(2000U, VD_3[0].Count);
955   ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
956   ASSERT_EQ(1800U, VD_3[1].Count);
957 }
958 
TEST_P(MaybeSparseInstrProfTest,value_prof_data_read_write_mapping)959 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
960 
961   NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
962   addValueProfData(SrcRecord);
963   std::unique_ptr<ValueProfData> VPData =
964       ValueProfData::serializeFrom(SrcRecord);
965 
966   NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
967   InstrProfSymtab Symtab;
968   Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);
969   Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);
970   Symtab.mapAddress(uint64_t(callee3), 0x3000ULL);
971   Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
972   // Missing mapping for callee5
973 
974   VPData->deserializeTo(Record, &Symtab);
975 
976   // Now read data from Record and sanity check the data
977   ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
978   ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
979 
980   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
981     return VD1.Count > VD2.Count;
982   };
983   std::unique_ptr<InstrProfValueData[]> VD_0(
984       Record.getValueForSite(IPVK_IndirectCallTarget, 0));
985   llvm::sort(&VD_0[0], &VD_0[5], Cmp);
986   ASSERT_EQ(VD_0[0].Value, 0x2000ULL);
987   ASSERT_EQ(1000U, VD_0[0].Count);
988   ASSERT_EQ(VD_0[1].Value, 0x3000ULL);
989   ASSERT_EQ(500U, VD_0[1].Count);
990   ASSERT_EQ(VD_0[2].Value, 0x1000ULL);
991   ASSERT_EQ(400U, VD_0[2].Count);
992 
993   // callee5 does not have a mapped value -- default to 0.
994   ASSERT_EQ(VD_0[4].Value, 0ULL);
995 }
996 
TEST_P(MaybeSparseInstrProfTest,get_max_function_count)997 TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
998   Writer.addRecord({"foo", 0x1234, {1ULL << 31, 2}}, Err);
999   Writer.addRecord({"bar", 0, {1ULL << 63}}, Err);
1000   Writer.addRecord({"baz", 0x5678, {0, 0, 0, 0}}, Err);
1001   auto Profile = Writer.writeBuffer();
1002   readProfile(std::move(Profile));
1003 
1004   ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));
1005 }
1006 
TEST_P(MaybeSparseInstrProfTest,get_weighted_function_counts)1007 TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {
1008   Writer.addRecord({"foo", 0x1234, {1, 2}}, 3, Err);
1009   Writer.addRecord({"foo", 0x1235, {3, 4}}, 5, Err);
1010   auto Profile = Writer.writeBuffer();
1011   readProfile(std::move(Profile));
1012 
1013   std::vector<uint64_t> Counts;
1014   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
1015                     Succeeded());
1016   ASSERT_EQ(2U, Counts.size());
1017   ASSERT_EQ(3U, Counts[0]);
1018   ASSERT_EQ(6U, Counts[1]);
1019 
1020   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
1021                     Succeeded());
1022   ASSERT_EQ(2U, Counts.size());
1023   ASSERT_EQ(15U, Counts[0]);
1024   ASSERT_EQ(20U, Counts[1]);
1025 }
1026 
1027 // Testing symtab creator interface used by indexed profile reader.
TEST_P(MaybeSparseInstrProfTest,instr_prof_symtab_test)1028 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_test) {
1029   std::vector<StringRef> FuncNames;
1030   FuncNames.push_back("func1");
1031   FuncNames.push_back("func2");
1032   FuncNames.push_back("func3");
1033   FuncNames.push_back("bar1");
1034   FuncNames.push_back("bar2");
1035   FuncNames.push_back("bar3");
1036   InstrProfSymtab Symtab;
1037   EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());
1038   StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1039   ASSERT_EQ(StringRef("func1"), R);
1040   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1041   ASSERT_EQ(StringRef("func2"), R);
1042   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1043   ASSERT_EQ(StringRef("func3"), R);
1044   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1045   ASSERT_EQ(StringRef("bar1"), R);
1046   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1047   ASSERT_EQ(StringRef("bar2"), R);
1048   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1049   ASSERT_EQ(StringRef("bar3"), R);
1050 
1051   // negative tests
1052   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar4"));
1053   ASSERT_EQ(StringRef(), R);
1054   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("foo4"));
1055   ASSERT_EQ(StringRef(), R);
1056 
1057   // Now incrementally update the symtab
1058   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());
1059   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());
1060   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());
1061 
1062   // Check again
1063   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_1"));
1064   ASSERT_EQ(StringRef("blah_1"), R);
1065   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_2"));
1066   ASSERT_EQ(StringRef("blah_2"), R);
1067   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_3"));
1068   ASSERT_EQ(StringRef("blah_3"), R);
1069   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1070   ASSERT_EQ(StringRef("func1"), R);
1071   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1072   ASSERT_EQ(StringRef("func2"), R);
1073   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1074   ASSERT_EQ(StringRef("func3"), R);
1075   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1076   ASSERT_EQ(StringRef("bar1"), R);
1077   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1078   ASSERT_EQ(StringRef("bar2"), R);
1079   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1080   ASSERT_EQ(StringRef("bar3"), R);
1081 }
1082 
1083 // Test that we get an error when creating a bogus symtab.
TEST_P(MaybeSparseInstrProfTest,instr_prof_bogus_symtab_empty_func_name)1084 TEST_P(MaybeSparseInstrProfTest, instr_prof_bogus_symtab_empty_func_name) {
1085   InstrProfSymtab Symtab;
1086   EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));
1087 }
1088 
1089 // Testing symtab creator interface used by value profile transformer.
TEST_P(MaybeSparseInstrProfTest,instr_prof_symtab_module_test)1090 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) {
1091   LLVMContext Ctx;
1092   std::unique_ptr<Module> M = std::make_unique<Module>("MyModule.cpp", Ctx);
1093   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
1094                                         /*isVarArg=*/false);
1095   Function::Create(FTy, Function::ExternalLinkage, "Gfoo", M.get());
1096   Function::Create(FTy, Function::ExternalLinkage, "Gblah", M.get());
1097   Function::Create(FTy, Function::ExternalLinkage, "Gbar", M.get());
1098   Function::Create(FTy, Function::InternalLinkage, "Ifoo", M.get());
1099   Function::Create(FTy, Function::InternalLinkage, "Iblah", M.get());
1100   Function::Create(FTy, Function::InternalLinkage, "Ibar", M.get());
1101   Function::Create(FTy, Function::PrivateLinkage, "Pfoo", M.get());
1102   Function::Create(FTy, Function::PrivateLinkage, "Pblah", M.get());
1103   Function::Create(FTy, Function::PrivateLinkage, "Pbar", M.get());
1104   Function::Create(FTy, Function::WeakODRLinkage, "Wfoo", M.get());
1105   Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());
1106   Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());
1107 
1108   InstrProfSymtab ProfSymtab;
1109   EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
1110 
1111   StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",
1112                        "Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};
1113 
1114   for (unsigned I = 0; I < sizeof(Funcs) / sizeof(*Funcs); I++) {
1115     Function *F = M->getFunction(Funcs[I]);
1116     ASSERT_TRUE(F != nullptr);
1117     std::string PGOName = getPGOFuncName(*F);
1118     uint64_t Key = IndexedInstrProf::ComputeHash(PGOName);
1119     ASSERT_EQ(StringRef(PGOName),
1120               ProfSymtab.getFuncName(Key));
1121     ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key));
1122   }
1123 }
1124 
1125 // Testing symtab serialization and creator/deserialization interface
1126 // used by coverage map reader, and raw profile reader.
TEST_P(MaybeSparseInstrProfTest,instr_prof_symtab_compression_test)1127 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_compression_test) {
1128   std::vector<std::string> FuncNames1;
1129   std::vector<std::string> FuncNames2;
1130   for (int I = 0; I < 3; I++) {
1131     std::string str;
1132     raw_string_ostream OS(str);
1133     OS << "func_" << I;
1134     FuncNames1.push_back(OS.str());
1135     str.clear();
1136     OS << "f oooooooooooooo_" << I;
1137     FuncNames1.push_back(OS.str());
1138     str.clear();
1139     OS << "BAR_" << I;
1140     FuncNames2.push_back(OS.str());
1141     str.clear();
1142     OS << "BlahblahBlahblahBar_" << I;
1143     FuncNames2.push_back(OS.str());
1144   }
1145 
1146   for (bool DoCompression : {false, true}) {
1147     // Compressing:
1148     std::string FuncNameStrings1;
1149     EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1150                           FuncNames1,
1151                           (DoCompression && compression::zlib::isAvailable()),
1152                           FuncNameStrings1),
1153                       Succeeded());
1154 
1155     // Compressing:
1156     std::string FuncNameStrings2;
1157     EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1158                           FuncNames2,
1159                           (DoCompression && compression::zlib::isAvailable()),
1160                           FuncNameStrings2),
1161                       Succeeded());
1162 
1163     for (int Padding = 0; Padding < 2; Padding++) {
1164       // Join with paddings :
1165       std::string FuncNameStrings = FuncNameStrings1;
1166       for (int P = 0; P < Padding; P++) {
1167         FuncNameStrings.push_back('\0');
1168       }
1169       FuncNameStrings += FuncNameStrings2;
1170 
1171       // Now decompress:
1172       InstrProfSymtab Symtab;
1173       EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());
1174 
1175       // Now do the checks:
1176       // First sampling some data points:
1177       StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[0]));
1178       ASSERT_EQ(StringRef("func_0"), R);
1179       R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[1]));
1180       ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);
1181       for (int I = 0; I < 3; I++) {
1182         std::string N[4];
1183         N[0] = FuncNames1[2 * I];
1184         N[1] = FuncNames1[2 * I + 1];
1185         N[2] = FuncNames2[2 * I];
1186         N[3] = FuncNames2[2 * I + 1];
1187         for (int J = 0; J < 4; J++) {
1188           StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(N[J]));
1189           ASSERT_EQ(StringRef(N[J]), R);
1190         }
1191       }
1192     }
1193   }
1194 }
1195 
TEST_P(MaybeSparseInstrProfTest,remapping_test)1196 TEST_P(MaybeSparseInstrProfTest, remapping_test) {
1197   Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);
1198   Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err);
1199   auto Profile = Writer.writeBuffer();
1200   readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(
1201     type i l
1202     name 3bar 4quux
1203   )"));
1204 
1205   std::vector<uint64_t> Counts;
1206   for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1207     EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1208                       Succeeded());
1209     ASSERT_EQ(4u, Counts.size());
1210     EXPECT_EQ(1u, Counts[0]);
1211     EXPECT_EQ(2u, Counts[1]);
1212     EXPECT_EQ(3u, Counts[2]);
1213     EXPECT_EQ(4u, Counts[3]);
1214   }
1215 
1216   for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) {
1217     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1218                       Succeeded());
1219     ASSERT_EQ(3u, Counts.size());
1220     EXPECT_EQ(5u, Counts[0]);
1221     EXPECT_EQ(6u, Counts[1]);
1222     EXPECT_EQ(7u, Counts[2]);
1223   }
1224 
1225   for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1226                             "_Z3barf", "otherfile:_Z4quuxf"}) {
1227     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1228                       Failed());
1229     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1230                       Failed());
1231   }
1232 }
1233 
TEST_F(SparseInstrProfTest,preserve_no_records)1234 TEST_F(SparseInstrProfTest, preserve_no_records) {
1235   Writer.addRecord({"foo", 0x1234, {0}}, Err);
1236   Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);
1237   Writer.addRecord({"baz", 0x4321, {0, 0, 0}}, Err);
1238 
1239   auto Profile = Writer.writeBuffer();
1240   readProfile(std::move(Profile));
1241 
1242   auto I = Reader->begin(), E = Reader->end();
1243   ASSERT_TRUE(I == E);
1244 }
1245 
1246 INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,
1247                          ::testing::Bool());
1248 
1249 #if defined(_LP64) && defined(EXPENSIVE_CHECKS)
TEST(ProfileReaderTest,ReadsLargeFiles)1250 TEST(ProfileReaderTest, ReadsLargeFiles) {
1251   const size_t LargeSize = 1ULL << 32; // 4GB
1252 
1253   auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1254   if (!RawProfile)
1255     return;
1256   auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
1257   ASSERT_TRUE(InstrProfError::take(RawProfileReaderOrErr.takeError()) ==
1258               instrprof_error::unrecognized_format);
1259 
1260   auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1261   if (!IndexedProfile)
1262     return;
1263   auto IndexedReaderOrErr =
1264       IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
1265   ASSERT_TRUE(InstrProfError::take(IndexedReaderOrErr.takeError()) ==
1266               instrprof_error::bad_magic);
1267 }
1268 #endif
1269 
1270 } // end anonymous namespace
1271