1 //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/IR/Function.h"
10 #include "llvm/IR/IRBuilder.h"
11 #include "llvm/IR/LLVMContext.h"
12 #include "llvm/IR/Module.h"
13 #include "llvm/ProfileData/InstrProfReader.h"
14 #include "llvm/ProfileData/InstrProfWriter.h"
15 #include "llvm/ProfileData/MemProf.h"
16 #include "llvm/ProfileData/MemProfData.inc"
17 #include "llvm/Support/Compression.h"
18 #include "llvm/Support/raw_ostream.h"
19 #include "llvm/Testing/Support/Error.h"
20 #include "llvm/Testing/Support/SupportHelpers.h"
21 #include "gtest/gtest.h"
22 #include <cstdarg>
23 
24 using namespace llvm;
25 
26 LLVM_NODISCARD static ::testing::AssertionResult
27 ErrorEquals(instrprof_error Expected, Error E) {
28   instrprof_error Found;
29   std::string FoundMsg;
30   handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
31     Found = IPE.get();
32     FoundMsg = IPE.message();
33   });
34   if (Expected == Found)
35     return ::testing::AssertionSuccess();
36   return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";
37 }
38 
39 namespace {
40 
41 struct InstrProfTest : ::testing::Test {
42   InstrProfWriter Writer;
43   std::unique_ptr<IndexedInstrProfReader> Reader;
44 
45   void SetUp() override { Writer.setOutputSparse(false); }
46 
47   void readProfile(std::unique_ptr<MemoryBuffer> Profile,
48                    std::unique_ptr<MemoryBuffer> Remapping = nullptr) {
49     auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),
50                                                       std::move(Remapping));
51     EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
52     Reader = std::move(ReaderOrErr.get());
53   }
54 };
55 
56 struct SparseInstrProfTest : public InstrProfTest {
57   void SetUp() override { Writer.setOutputSparse(true); }
58 };
59 
60 struct MaybeSparseInstrProfTest : public InstrProfTest,
61                                   public ::testing::WithParamInterface<bool> {
62   void SetUp() override { Writer.setOutputSparse(GetParam()); }
63 };
64 
65 TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
66   auto Profile = Writer.writeBuffer();
67   readProfile(std::move(Profile));
68   ASSERT_TRUE(Reader->begin() == Reader->end());
69 }
70 
71 static const auto Err = [](Error E) {
72   consumeError(std::move(E));
73   FAIL();
74 };
75 
76 TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
77   Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
78   auto Profile = Writer.writeBuffer();
79   readProfile(std::move(Profile));
80 
81   auto I = Reader->begin(), E = Reader->end();
82   ASSERT_TRUE(I != E);
83   ASSERT_EQ(StringRef("foo"), I->Name);
84   ASSERT_EQ(0x1234U, I->Hash);
85   ASSERT_EQ(4U, I->Counts.size());
86   ASSERT_EQ(1U, I->Counts[0]);
87   ASSERT_EQ(2U, I->Counts[1]);
88   ASSERT_EQ(3U, I->Counts[2]);
89   ASSERT_EQ(4U, I->Counts[3]);
90   ASSERT_TRUE(++I == E);
91 }
92 
93 TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
94   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
95   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
96   auto Profile = Writer.writeBuffer();
97   readProfile(std::move(Profile));
98 
99   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("foo", 0x1234);
100   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
101   ASSERT_EQ(2U, R->Counts.size());
102   ASSERT_EQ(1U, R->Counts[0]);
103   ASSERT_EQ(2U, R->Counts[1]);
104 
105   R = Reader->getInstrProfRecord("foo", 0x1235);
106   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
107   ASSERT_EQ(2U, R->Counts.size());
108   ASSERT_EQ(3U, R->Counts[0]);
109   ASSERT_EQ(4U, R->Counts[1]);
110 
111   R = Reader->getInstrProfRecord("foo", 0x5678);
112   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));
113 
114   R = Reader->getInstrProfRecord("bar", 0x1234);
115   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));
116 }
117 
118 TEST_P(MaybeSparseInstrProfTest, get_function_counts) {
119   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
120   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
121   auto Profile = Writer.writeBuffer();
122   readProfile(std::move(Profile));
123 
124   std::vector<uint64_t> Counts;
125   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
126                     Succeeded());
127   ASSERT_EQ(2U, Counts.size());
128   ASSERT_EQ(1U, Counts[0]);
129   ASSERT_EQ(2U, Counts[1]);
130 
131   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
132                     Succeeded());
133   ASSERT_EQ(2U, Counts.size());
134   ASSERT_EQ(3U, Counts[0]);
135   ASSERT_EQ(4U, Counts[1]);
136 
137   Error E1 = Reader->getFunctionCounts("foo", 0x5678, Counts);
138   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));
139 
140   Error E2 = Reader->getFunctionCounts("bar", 0x1234, Counts);
141   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));
142 }
143 
144 // Profile data is copied from general.proftext
145 TEST_F(InstrProfTest, get_profile_summary) {
146   Writer.addRecord({"func1", 0x1234, {97531}}, Err);
147   Writer.addRecord({"func2", 0x1234, {0, 0}}, Err);
148   Writer.addRecord(
149       {"func3",
150        0x1234,
151        {2305843009213693952, 1152921504606846976, 576460752303423488,
152         288230376151711744, 144115188075855872, 72057594037927936}},
153       Err);
154   Writer.addRecord({"func4", 0x1234, {0}}, Err);
155   auto Profile = Writer.writeBuffer();
156   readProfile(std::move(Profile));
157 
158   auto VerifySummary = [](ProfileSummary &IPS) mutable {
159     ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());
160     ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());
161     ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());
162     ASSERT_EQ(10U, IPS.getNumCounts());
163     ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());
164     const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();
165     uint32_t Cutoff = 800000;
166     auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
167       return PE.Cutoff == Cutoff;
168     };
169     auto EightyPerc = find_if(Details, Predicate);
170     Cutoff = 900000;
171     auto NinetyPerc = find_if(Details, Predicate);
172     Cutoff = 950000;
173     auto NinetyFivePerc = find_if(Details, Predicate);
174     Cutoff = 990000;
175     auto NinetyNinePerc = find_if(Details, Predicate);
176     ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);
177     ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);
178     ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);
179     ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);
180   };
181   ProfileSummary &PS = Reader->getSummary(/* IsCS */ false);
182   VerifySummary(PS);
183 
184   // Test that conversion of summary to and from Metadata works.
185   LLVMContext Context;
186   Metadata *MD = PS.getMD(Context);
187   ASSERT_TRUE(MD);
188   ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);
189   ASSERT_TRUE(PSFromMD);
190   VerifySummary(*PSFromMD);
191   delete PSFromMD;
192 
193   // Test that summary can be attached to and read back from module.
194   Module M("my_module", Context);
195   M.setProfileSummary(MD, ProfileSummary::PSK_Instr);
196   MD = M.getProfileSummary(/* IsCS */ false);
197   ASSERT_TRUE(MD);
198   PSFromMD = ProfileSummary::getFromMD(MD);
199   ASSERT_TRUE(PSFromMD);
200   VerifySummary(*PSFromMD);
201   delete PSFromMD;
202 }
203 
204 TEST_F(InstrProfTest, test_writer_merge) {
205   Writer.addRecord({"func1", 0x1234, {42}}, Err);
206 
207   InstrProfWriter Writer2;
208   Writer2.addRecord({"func2", 0x1234, {0, 0}}, Err);
209 
210   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
211 
212   auto Profile = Writer.writeBuffer();
213   readProfile(std::move(Profile));
214 
215   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
216   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
217   ASSERT_EQ(1U, R->Counts.size());
218   ASSERT_EQ(42U, R->Counts[0]);
219 
220   R = Reader->getInstrProfRecord("func2", 0x1234);
221   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
222   ASSERT_EQ(2U, R->Counts.size());
223   ASSERT_EQ(0U, R->Counts[0]);
224   ASSERT_EQ(0U, R->Counts[1]);
225 }
226 
227 using ::llvm::memprof::IndexedMemProfRecord;
228 using ::llvm::memprof::MemInfoBlock;
229 using FrameIdMapTy =
230     llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
231 
232 static FrameIdMapTy getFrameMapping() {
233   FrameIdMapTy Mapping;
234   Mapping.insert({0, {0x123, 1, 2, false}});
235   Mapping.insert({1, {0x345, 3, 4, true}});
236   Mapping.insert({2, {0x125, 5, 6, false}});
237   Mapping.insert({3, {0x567, 7, 8, true}});
238   Mapping.insert({4, {0x124, 5, 6, false}});
239   Mapping.insert({5, {0x789, 8, 9, true}});
240   return Mapping;
241 }
242 
243 IndexedMemProfRecord makeRecord(
244     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
245         AllocFrames,
246     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
247         CallSiteFrames,
248     const MemInfoBlock &Block = MemInfoBlock()) {
249   llvm::memprof::IndexedMemProfRecord MR;
250   for (const auto &Frames : AllocFrames)
251     MR.AllocSites.emplace_back(Frames, Block);
252   for (const auto &Frames : CallSiteFrames)
253     MR.CallSites.push_back(Frames);
254   return MR;
255 }
256 
257 MATCHER_P(EqualsRecord, Want, "") {
258   const memprof::MemProfRecord &Got = arg;
259 
260   auto PrintAndFail = [&]() {
261     std::string Buffer;
262     llvm::raw_string_ostream OS(Buffer);
263     OS << "Want:\n";
264     Want.print(OS);
265     OS << "Got:\n";
266     Got.print(OS);
267     OS.flush();
268     *result_listener << "MemProf Record differs!\n" << Buffer;
269     return false;
270   };
271 
272   if (Want.AllocSites.size() != Got.AllocSites.size())
273     return PrintAndFail();
274   if (Want.CallSites.size() != Got.CallSites.size())
275     return PrintAndFail();
276 
277   for (size_t I = 0; I < Got.AllocSites.size(); I++) {
278     if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)
279       return PrintAndFail();
280     if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)
281       return PrintAndFail();
282   }
283 
284   for (size_t I = 0; I < Got.CallSites.size(); I++) {
285     if (Want.CallSites[I] != Got.CallSites[I])
286       return PrintAndFail();
287   }
288   return true;
289 }
290 
291 TEST_F(InstrProfTest, test_memprof) {
292   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
293                     Succeeded());
294 
295   const IndexedMemProfRecord IndexedMR = makeRecord(
296       /*AllocFrames=*/
297       {
298           {0, 1},
299           {2, 3},
300       },
301       /*CallSiteFrames=*/{
302           {4, 5},
303       });
304   const FrameIdMapTy IdToFrameMap = getFrameMapping();
305   for (const auto &I : IdToFrameMap) {
306     Writer.addMemProfFrame(I.first, I.getSecond(), Err);
307   }
308   Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
309 
310   auto Profile = Writer.writeBuffer();
311   readProfile(std::move(Profile));
312 
313   auto RecordOr = Reader->getMemProfRecord(0x9999);
314   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
315   const memprof::MemProfRecord &Record = RecordOr.get();
316 
317   memprof::FrameId LastUnmappedFrameId = 0;
318   bool HasFrameMappingError = false;
319   auto IdToFrameCallback = [&](const memprof::FrameId Id) {
320     auto Iter = IdToFrameMap.find(Id);
321     if (Iter == IdToFrameMap.end()) {
322       LastUnmappedFrameId = Id;
323       HasFrameMappingError = true;
324       return memprof::Frame(0, 0, 0, false);
325     }
326     return Iter->second;
327   };
328 
329   const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
330   ASSERT_FALSE(HasFrameMappingError)
331       << "could not map frame id: " << LastUnmappedFrameId;
332   EXPECT_THAT(WantRecord, EqualsRecord(Record));
333 }
334 
335 TEST_F(InstrProfTest, test_memprof_getrecord_error) {
336   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
337                     Succeeded());
338 
339   const IndexedMemProfRecord IndexedMR = makeRecord(
340       /*AllocFrames=*/
341       {
342           {0, 1},
343           {2, 3},
344       },
345       /*CallSiteFrames=*/{
346           {4, 5},
347       });
348   // We skip adding the frame mappings here unlike the test_memprof unit test
349   // above to exercise the failure path when getMemProfRecord is invoked.
350   Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
351 
352   auto Profile = Writer.writeBuffer();
353   readProfile(std::move(Profile));
354 
355   auto RecordOr = Reader->getMemProfRecord(0x9999);
356   EXPECT_THAT_ERROR(RecordOr.takeError(), Failed());
357 }
358 
359 TEST_F(InstrProfTest, test_memprof_merge) {
360   Writer.addRecord({"func1", 0x1234, {42}}, Err);
361 
362   InstrProfWriter Writer2;
363   ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),
364                     Succeeded());
365 
366   const IndexedMemProfRecord IndexedMR = makeRecord(
367       /*AllocFrames=*/
368       {
369           {0, 1},
370           {2, 3},
371       },
372       /*CallSiteFrames=*/{
373           {4, 5},
374       });
375 
376   const FrameIdMapTy IdToFrameMap = getFrameMapping();
377   for (const auto &I : IdToFrameMap) {
378     Writer.addMemProfFrame(I.first, I.getSecond(), Err);
379   }
380   Writer2.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
381 
382   ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),
383                     Succeeded());
384   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
385 
386   auto Profile = Writer.writeBuffer();
387   readProfile(std::move(Profile));
388 
389   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
390   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
391   ASSERT_EQ(1U, R->Counts.size());
392   ASSERT_EQ(42U, R->Counts[0]);
393 
394   auto RecordOr = Reader->getMemProfRecord(0x9999);
395   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
396   const memprof::MemProfRecord &Record = RecordOr.get();
397 
398   memprof::FrameId LastUnmappedFrameId = 0;
399   bool HasFrameMappingError = false;
400 
401   auto IdToFrameCallback = [&](const memprof::FrameId Id) {
402     auto Iter = IdToFrameMap.find(Id);
403     if (Iter == IdToFrameMap.end()) {
404       LastUnmappedFrameId = Id;
405       HasFrameMappingError = true;
406       return memprof::Frame(0, 0, 0, false);
407     }
408     return Iter->second;
409   };
410 
411   const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
412   ASSERT_FALSE(HasFrameMappingError)
413       << "could not map frame id: " << LastUnmappedFrameId;
414   EXPECT_THAT(WantRecord, EqualsRecord(Record));
415 }
416 
417 static const char callee1[] = "callee1";
418 static const char callee2[] = "callee2";
419 static const char callee3[] = "callee3";
420 static const char callee4[] = "callee4";
421 static const char callee5[] = "callee5";
422 static const char callee6[] = "callee6";
423 
424 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {
425   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
426 
427   // 4 value sites.
428   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
429   InstrProfValueData VD0[] = {
430       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
431   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
432   // No value profile data at the second site.
433   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
434   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
435   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
436   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
437   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
438 
439   Writer.addRecord(std::move(Record1), Err);
440   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
441   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
442   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
443   auto Profile = Writer.writeBuffer();
444   readProfile(std::move(Profile));
445 
446   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
447   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
448   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
449   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
450   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
451   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
452   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
453 
454   uint64_t TotalC;
455   std::unique_ptr<InstrProfValueData[]> VD =
456       R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
457 
458   ASSERT_EQ(3U, VD[0].Count);
459   ASSERT_EQ(2U, VD[1].Count);
460   ASSERT_EQ(1U, VD[2].Count);
461   ASSERT_EQ(6U, TotalC);
462 
463   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
464   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
465   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
466 }
467 
468 TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
469   NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
470   Record.reserveSites(IPVK_IndirectCallTarget, 1);
471   InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
472                               {4000, 4}, {6000, 6}};
473   Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 6, nullptr);
474   Writer.addRecord(std::move(Record), Err);
475   auto Profile = Writer.writeBuffer();
476   readProfile(std::move(Profile));
477   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
478   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
479 
480   LLVMContext Ctx;
481   std::unique_ptr<Module> M(new Module("MyModule", Ctx));
482   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
483                                         /*isVarArg=*/false);
484   Function *F =
485       Function::Create(FTy, Function::ExternalLinkage, "caller", M.get());
486   BasicBlock *BB = BasicBlock::Create(Ctx, "", F);
487 
488   IRBuilder<> Builder(BB);
489   BasicBlock *TBB = BasicBlock::Create(Ctx, "", F);
490   BasicBlock *FBB = BasicBlock::Create(Ctx, "", F);
491 
492   // Use branch instruction to annotate with value profile data for simplicity
493   Instruction *Inst = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
494   Instruction *Inst2 = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
495   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0);
496 
497   InstrProfValueData ValueData[5];
498   uint32_t N;
499   uint64_t T;
500   bool Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
501                                       ValueData, N, T);
502   ASSERT_TRUE(Res);
503   ASSERT_EQ(3U, N);
504   ASSERT_EQ(21U, T);
505   // The result should be sorted already:
506   ASSERT_EQ(6000U, ValueData[0].Value);
507   ASSERT_EQ(6U, ValueData[0].Count);
508   ASSERT_EQ(5000U, ValueData[1].Value);
509   ASSERT_EQ(5U, ValueData[1].Count);
510   ASSERT_EQ(4000U, ValueData[2].Value);
511   ASSERT_EQ(4U, ValueData[2].Count);
512   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 1, ValueData,
513                                  N, T);
514   ASSERT_TRUE(Res);
515   ASSERT_EQ(1U, N);
516   ASSERT_EQ(21U, T);
517 
518   Res = getValueProfDataFromInst(*Inst2, IPVK_IndirectCallTarget, 5, ValueData,
519                                  N, T);
520   ASSERT_FALSE(Res);
521 
522   // Remove the MD_prof metadata
523   Inst->setMetadata(LLVMContext::MD_prof, 0);
524   // Annotate 5 records this time.
525   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);
526   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
527                                       ValueData, N, T);
528   ASSERT_TRUE(Res);
529   ASSERT_EQ(5U, N);
530   ASSERT_EQ(21U, T);
531   ASSERT_EQ(6000U, ValueData[0].Value);
532   ASSERT_EQ(6U, ValueData[0].Count);
533   ASSERT_EQ(5000U, ValueData[1].Value);
534   ASSERT_EQ(5U, ValueData[1].Count);
535   ASSERT_EQ(4000U, ValueData[2].Value);
536   ASSERT_EQ(4U, ValueData[2].Count);
537   ASSERT_EQ(3000U, ValueData[3].Value);
538   ASSERT_EQ(3U, ValueData[3].Count);
539   ASSERT_EQ(2000U, ValueData[4].Value);
540   ASSERT_EQ(2U, ValueData[4].Count);
541 
542   // Remove the MD_prof metadata
543   Inst->setMetadata(LLVMContext::MD_prof, 0);
544   // Annotate with 4 records.
545   InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},
546                               {5000, 2}, {6000, 1}};
547   annotateValueSite(*M, *Inst, makeArrayRef(VD0Sorted).slice(2), 10,
548                     IPVK_IndirectCallTarget, 5);
549   Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
550                                       ValueData, N, T);
551   ASSERT_TRUE(Res);
552   ASSERT_EQ(4U, N);
553   ASSERT_EQ(10U, T);
554   ASSERT_EQ(3000U, ValueData[0].Value);
555   ASSERT_EQ(4U, ValueData[0].Count);
556   ASSERT_EQ(4000U, ValueData[1].Value);
557   ASSERT_EQ(3U, ValueData[1].Count);
558   ASSERT_EQ(5000U, ValueData[2].Value);
559   ASSERT_EQ(2U, ValueData[2].Count);
560   ASSERT_EQ(6000U, ValueData[3].Value);
561   ASSERT_EQ(1U, ValueData[3].Count);
562 }
563 
564 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) {
565   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
566 
567   // 4 value sites.
568   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
569   InstrProfValueData VD0[] = {
570       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
571   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
572   // No value profile data at the second site.
573   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
574   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
575   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
576   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
577   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
578 
579   Writer.addRecord(std::move(Record1), 10, Err);
580   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
581   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
582   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
583   auto Profile = Writer.writeBuffer();
584   readProfile(std::move(Profile));
585 
586   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
587   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
588   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
589   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
590   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
591   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
592   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
593 
594   uint64_t TotalC;
595   std::unique_ptr<InstrProfValueData[]> VD =
596       R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
597   ASSERT_EQ(30U, VD[0].Count);
598   ASSERT_EQ(20U, VD[1].Count);
599   ASSERT_EQ(10U, VD[2].Count);
600   ASSERT_EQ(60U, TotalC);
601 
602   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
603   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
604   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
605 }
606 
607 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
608   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
609 
610   // 4 value sites.
611   Record1.reserveSites(IPVK_IndirectCallTarget, 4);
612   InstrProfValueData VD0[] = {
613       {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
614   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
615   // No value profile data at the second site.
616   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
617   InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
618   Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
619   InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
620   Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
621 
622   Writer.addRecord(std::move(Record1), Err);
623   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
624   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
625   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
626 
627   // Set big endian output.
628   Writer.setValueProfDataEndianness(support::big);
629 
630   auto Profile = Writer.writeBuffer();
631   readProfile(std::move(Profile));
632 
633   // Set big endian input.
634   Reader->setValueProfDataEndianness(support::big);
635 
636   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
637   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
638   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
639   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
640   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
641   ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
642   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
643 
644   std::unique_ptr<InstrProfValueData[]> VD =
645       R->getValueForSite(IPVK_IndirectCallTarget, 0);
646   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
647   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
648   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
649 
650   // Restore little endian default:
651   Writer.setValueProfDataEndianness(support::little);
652 }
653 
654 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
655   static const char caller[] = "caller";
656   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
657   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
658 
659   // 5 value sites.
660   Record11.reserveSites(IPVK_IndirectCallTarget, 5);
661   InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
662                               {uint64_t(callee2), 2},
663                               {uint64_t(callee3), 3},
664                               {uint64_t(callee4), 4}};
665   Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
666 
667   // No value profile data at the second site.
668   Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
669 
670   InstrProfValueData VD2[] = {
671       {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
672   Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
673 
674   InstrProfValueData VD3[] = {{uint64_t(callee1), 1}};
675   Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
676 
677   InstrProfValueData VD4[] = {{uint64_t(callee1), 1},
678                               {uint64_t(callee2), 2},
679                               {uint64_t(callee3), 3}};
680   Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
681 
682   // A different record for the same caller.
683   Record12.reserveSites(IPVK_IndirectCallTarget, 5);
684   InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};
685   Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, 2, nullptr);
686 
687   // No value profile data at the second site.
688   Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
689 
690   InstrProfValueData VD22[] = {
691       {uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};
692   Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, 3, nullptr);
693 
694   Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr);
695 
696   InstrProfValueData VD42[] = {{uint64_t(callee1), 1},
697                                {uint64_t(callee2), 2},
698                                {uint64_t(callee3), 3}};
699   Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr);
700 
701   Writer.addRecord(std::move(Record11), Err);
702   // Merge profile data.
703   Writer.addRecord(std::move(Record12), Err);
704 
705   Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);
706   Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);
707   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
708   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
709   Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
710   auto Profile = Writer.writeBuffer();
711   readProfile(std::move(Profile));
712 
713   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
714   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
715   ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
716   ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
717   ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
718   ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
719   ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
720   ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
721 
722   std::unique_ptr<InstrProfValueData[]> VD =
723       R->getValueForSite(IPVK_IndirectCallTarget, 0);
724   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
725   ASSERT_EQ(7U, VD[0].Count);
726   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
727   ASSERT_EQ(6U, VD[1].Count);
728   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
729   ASSERT_EQ(4U, VD[2].Count);
730   ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
731   ASSERT_EQ(1U, VD[3].Count);
732 
733   std::unique_ptr<InstrProfValueData[]> VD_2(
734       R->getValueForSite(IPVK_IndirectCallTarget, 2));
735   ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
736   ASSERT_EQ(6U, VD_2[0].Count);
737   ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
738   ASSERT_EQ(4U, VD_2[1].Count);
739   ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
740   ASSERT_EQ(3U, VD_2[2].Count);
741   ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
742   ASSERT_EQ(1U, VD_2[3].Count);
743 
744   std::unique_ptr<InstrProfValueData[]> VD_3(
745       R->getValueForSite(IPVK_IndirectCallTarget, 3));
746   ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1"));
747   ASSERT_EQ(1U, VD_3[0].Count);
748 
749   std::unique_ptr<InstrProfValueData[]> VD_4(
750       R->getValueForSite(IPVK_IndirectCallTarget, 4));
751   ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
752   ASSERT_EQ(6U, VD_4[0].Count);
753   ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
754   ASSERT_EQ(4U, VD_4[1].Count);
755   ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
756   ASSERT_EQ(2U, VD_4[2].Count);
757 }
758 
759 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
760   static const char bar[] = "bar";
761 
762   const uint64_t Max = std::numeric_limits<uint64_t>::max();
763 
764   instrprof_error Result;
765   auto Err = [&](Error E) { Result = InstrProfError::take(std::move(E)); };
766   Result = instrprof_error::success;
767   Writer.addRecord({"foo", 0x1234, {1}}, Err);
768   ASSERT_EQ(Result, instrprof_error::success);
769 
770   // Verify counter overflow.
771   Result = instrprof_error::success;
772   Writer.addRecord({"foo", 0x1234, {Max}}, Err);
773   ASSERT_EQ(Result, instrprof_error::counter_overflow);
774 
775   Result = instrprof_error::success;
776   Writer.addRecord({bar, 0x9012, {8}}, Err);
777   ASSERT_EQ(Result, instrprof_error::success);
778 
779   NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
780   Record4.reserveSites(IPVK_IndirectCallTarget, 1);
781   InstrProfValueData VD4[] = {{uint64_t(bar), 1}};
782   Record4.addValueData(IPVK_IndirectCallTarget, 0, VD4, 1, nullptr);
783   Result = instrprof_error::success;
784   Writer.addRecord(std::move(Record4), Err);
785   ASSERT_EQ(Result, instrprof_error::success);
786 
787   // Verify value data counter overflow.
788   NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
789   Record5.reserveSites(IPVK_IndirectCallTarget, 1);
790   InstrProfValueData VD5[] = {{uint64_t(bar), Max}};
791   Record5.addValueData(IPVK_IndirectCallTarget, 0, VD5, 1, nullptr);
792   Result = instrprof_error::success;
793   Writer.addRecord(std::move(Record5), Err);
794   ASSERT_EQ(Result, instrprof_error::counter_overflow);
795 
796   auto Profile = Writer.writeBuffer();
797   readProfile(std::move(Profile));
798 
799   // Verify saturation of counts.
800   Expected<InstrProfRecord> ReadRecord1 =
801       Reader->getInstrProfRecord("foo", 0x1234);
802   EXPECT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
803   ASSERT_EQ(Max, ReadRecord1->Counts[0]);
804 
805   Expected<InstrProfRecord> ReadRecord2 =
806       Reader->getInstrProfRecord("baz", 0x5678);
807   ASSERT_TRUE(bool(ReadRecord2));
808   ASSERT_EQ(1U, ReadRecord2->getNumValueSites(IPVK_IndirectCallTarget));
809   std::unique_ptr<InstrProfValueData[]> VD =
810       ReadRecord2->getValueForSite(IPVK_IndirectCallTarget, 0);
811   ASSERT_EQ(StringRef("bar"), StringRef((const char *)VD[0].Value, 3));
812   ASSERT_EQ(Max, VD[0].Count);
813 }
814 
815 // This test tests that when there are too many values
816 // for a given site, the merged results are properly
817 // truncated.
818 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
819   static const char caller[] = "caller";
820 
821   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
822   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
823 
824   // 2 value sites.
825   Record11.reserveSites(IPVK_IndirectCallTarget, 2);
826   InstrProfValueData VD0[255];
827   for (int I = 0; I < 255; I++) {
828     VD0[I].Value = 2 * I;
829     VD0[I].Count = 2 * I + 1000;
830   }
831 
832   Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 255, nullptr);
833   Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
834 
835   Record12.reserveSites(IPVK_IndirectCallTarget, 2);
836   InstrProfValueData VD1[255];
837   for (int I = 0; I < 255; I++) {
838     VD1[I].Value = 2 * I + 1;
839     VD1[I].Count = 2 * I + 1001;
840   }
841 
842   Record12.addValueData(IPVK_IndirectCallTarget, 0, VD1, 255, nullptr);
843   Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
844 
845   Writer.addRecord(std::move(Record11), Err);
846   // Merge profile data.
847   Writer.addRecord(std::move(Record12), Err);
848 
849   auto Profile = Writer.writeBuffer();
850   readProfile(std::move(Profile));
851 
852   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
853   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
854   std::unique_ptr<InstrProfValueData[]> VD(
855       R->getValueForSite(IPVK_IndirectCallTarget, 0));
856   ASSERT_EQ(2U, R->getNumValueSites(IPVK_IndirectCallTarget));
857   ASSERT_EQ(255U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
858   for (unsigned I = 0; I < 255; I++) {
859     ASSERT_EQ(VD[I].Value, 509 - I);
860     ASSERT_EQ(VD[I].Count, 1509 - I);
861   }
862 }
863 
864 static void addValueProfData(InstrProfRecord &Record) {
865   Record.reserveSites(IPVK_IndirectCallTarget, 5);
866   InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
867                               {uint64_t(callee2), 1000},
868                               {uint64_t(callee3), 500},
869                               {uint64_t(callee4), 300},
870                               {uint64_t(callee5), 100}};
871   Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
872   InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
873                               {uint64_t(callee3), 1000},
874                               {uint64_t(callee2), 2500},
875                               {uint64_t(callee1), 1300}};
876   Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
877   InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
878                               {uint64_t(callee3), 1000},
879                               {uint64_t(callee4), 5500}};
880   Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
881   InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
882                               {uint64_t(callee3), 2000}};
883   Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
884   Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
885 }
886 
887 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) {
888   InstrProfRecord SrcRecord({1ULL << 31, 2});
889   addValueProfData(SrcRecord);
890   std::unique_ptr<ValueProfData> VPData =
891       ValueProfData::serializeFrom(SrcRecord);
892 
893   InstrProfRecord Record({1ULL << 31, 2});
894   VPData->deserializeTo(Record, nullptr);
895 
896   // Now read data from Record and sanity check the data
897   ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
898   ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
899   ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
900   ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
901   ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
902   ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
903 
904   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
905     return VD1.Count > VD2.Count;
906   };
907   std::unique_ptr<InstrProfValueData[]> VD_0(
908       Record.getValueForSite(IPVK_IndirectCallTarget, 0));
909   llvm::sort(&VD_0[0], &VD_0[5], Cmp);
910   ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
911   ASSERT_EQ(1000U, VD_0[0].Count);
912   ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
913   ASSERT_EQ(500U, VD_0[1].Count);
914   ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
915   ASSERT_EQ(400U, VD_0[2].Count);
916   ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
917   ASSERT_EQ(300U, VD_0[3].Count);
918   ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
919   ASSERT_EQ(100U, VD_0[4].Count);
920 
921   std::unique_ptr<InstrProfValueData[]> VD_1(
922       Record.getValueForSite(IPVK_IndirectCallTarget, 1));
923   llvm::sort(&VD_1[0], &VD_1[4], Cmp);
924   ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
925   ASSERT_EQ(2500U, VD_1[0].Count);
926   ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
927   ASSERT_EQ(1300U, VD_1[1].Count);
928   ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
929   ASSERT_EQ(1000U, VD_1[2].Count);
930   ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
931   ASSERT_EQ(800U, VD_1[3].Count);
932 
933   std::unique_ptr<InstrProfValueData[]> VD_2(
934       Record.getValueForSite(IPVK_IndirectCallTarget, 2));
935   llvm::sort(&VD_2[0], &VD_2[3], Cmp);
936   ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
937   ASSERT_EQ(5500U, VD_2[0].Count);
938   ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
939   ASSERT_EQ(1000U, VD_2[1].Count);
940   ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
941   ASSERT_EQ(800U, VD_2[2].Count);
942 
943   std::unique_ptr<InstrProfValueData[]> VD_3(
944       Record.getValueForSite(IPVK_IndirectCallTarget, 3));
945   llvm::sort(&VD_3[0], &VD_3[2], Cmp);
946   ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
947   ASSERT_EQ(2000U, VD_3[0].Count);
948   ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
949   ASSERT_EQ(1800U, VD_3[1].Count);
950 }
951 
952 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
953 
954   NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
955   addValueProfData(SrcRecord);
956   std::unique_ptr<ValueProfData> VPData =
957       ValueProfData::serializeFrom(SrcRecord);
958 
959   NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
960   InstrProfSymtab Symtab;
961   Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);
962   Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);
963   Symtab.mapAddress(uint64_t(callee3), 0x3000ULL);
964   Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
965   // Missing mapping for callee5
966 
967   VPData->deserializeTo(Record, &Symtab);
968 
969   // Now read data from Record and sanity check the data
970   ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
971   ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
972 
973   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
974     return VD1.Count > VD2.Count;
975   };
976   std::unique_ptr<InstrProfValueData[]> VD_0(
977       Record.getValueForSite(IPVK_IndirectCallTarget, 0));
978   llvm::sort(&VD_0[0], &VD_0[5], Cmp);
979   ASSERT_EQ(VD_0[0].Value, 0x2000ULL);
980   ASSERT_EQ(1000U, VD_0[0].Count);
981   ASSERT_EQ(VD_0[1].Value, 0x3000ULL);
982   ASSERT_EQ(500U, VD_0[1].Count);
983   ASSERT_EQ(VD_0[2].Value, 0x1000ULL);
984   ASSERT_EQ(400U, VD_0[2].Count);
985 
986   // callee5 does not have a mapped value -- default to 0.
987   ASSERT_EQ(VD_0[4].Value, 0ULL);
988 }
989 
990 TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
991   Writer.addRecord({"foo", 0x1234, {1ULL << 31, 2}}, Err);
992   Writer.addRecord({"bar", 0, {1ULL << 63}}, Err);
993   Writer.addRecord({"baz", 0x5678, {0, 0, 0, 0}}, Err);
994   auto Profile = Writer.writeBuffer();
995   readProfile(std::move(Profile));
996 
997   ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));
998 }
999 
1000 TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {
1001   Writer.addRecord({"foo", 0x1234, {1, 2}}, 3, Err);
1002   Writer.addRecord({"foo", 0x1235, {3, 4}}, 5, Err);
1003   auto Profile = Writer.writeBuffer();
1004   readProfile(std::move(Profile));
1005 
1006   std::vector<uint64_t> Counts;
1007   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
1008                     Succeeded());
1009   ASSERT_EQ(2U, Counts.size());
1010   ASSERT_EQ(3U, Counts[0]);
1011   ASSERT_EQ(6U, Counts[1]);
1012 
1013   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
1014                     Succeeded());
1015   ASSERT_EQ(2U, Counts.size());
1016   ASSERT_EQ(15U, Counts[0]);
1017   ASSERT_EQ(20U, Counts[1]);
1018 }
1019 
1020 // Testing symtab creator interface used by indexed profile reader.
1021 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_test) {
1022   std::vector<StringRef> FuncNames;
1023   FuncNames.push_back("func1");
1024   FuncNames.push_back("func2");
1025   FuncNames.push_back("func3");
1026   FuncNames.push_back("bar1");
1027   FuncNames.push_back("bar2");
1028   FuncNames.push_back("bar3");
1029   InstrProfSymtab Symtab;
1030   EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());
1031   StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1032   ASSERT_EQ(StringRef("func1"), R);
1033   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1034   ASSERT_EQ(StringRef("func2"), R);
1035   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1036   ASSERT_EQ(StringRef("func3"), R);
1037   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1038   ASSERT_EQ(StringRef("bar1"), R);
1039   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1040   ASSERT_EQ(StringRef("bar2"), R);
1041   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1042   ASSERT_EQ(StringRef("bar3"), R);
1043 
1044   // negative tests
1045   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar4"));
1046   ASSERT_EQ(StringRef(), R);
1047   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("foo4"));
1048   ASSERT_EQ(StringRef(), R);
1049 
1050   // Now incrementally update the symtab
1051   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());
1052   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());
1053   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());
1054 
1055   // Check again
1056   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_1"));
1057   ASSERT_EQ(StringRef("blah_1"), R);
1058   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_2"));
1059   ASSERT_EQ(StringRef("blah_2"), R);
1060   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_3"));
1061   ASSERT_EQ(StringRef("blah_3"), R);
1062   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1063   ASSERT_EQ(StringRef("func1"), R);
1064   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1065   ASSERT_EQ(StringRef("func2"), R);
1066   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1067   ASSERT_EQ(StringRef("func3"), R);
1068   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1069   ASSERT_EQ(StringRef("bar1"), R);
1070   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1071   ASSERT_EQ(StringRef("bar2"), R);
1072   R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1073   ASSERT_EQ(StringRef("bar3"), R);
1074 }
1075 
1076 // Test that we get an error when creating a bogus symtab.
1077 TEST_P(MaybeSparseInstrProfTest, instr_prof_bogus_symtab_empty_func_name) {
1078   InstrProfSymtab Symtab;
1079   EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));
1080 }
1081 
1082 // Testing symtab creator interface used by value profile transformer.
1083 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) {
1084   LLVMContext Ctx;
1085   std::unique_ptr<Module> M = std::make_unique<Module>("MyModule.cpp", Ctx);
1086   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
1087                                         /*isVarArg=*/false);
1088   Function::Create(FTy, Function::ExternalLinkage, "Gfoo", M.get());
1089   Function::Create(FTy, Function::ExternalLinkage, "Gblah", M.get());
1090   Function::Create(FTy, Function::ExternalLinkage, "Gbar", M.get());
1091   Function::Create(FTy, Function::InternalLinkage, "Ifoo", M.get());
1092   Function::Create(FTy, Function::InternalLinkage, "Iblah", M.get());
1093   Function::Create(FTy, Function::InternalLinkage, "Ibar", M.get());
1094   Function::Create(FTy, Function::PrivateLinkage, "Pfoo", M.get());
1095   Function::Create(FTy, Function::PrivateLinkage, "Pblah", M.get());
1096   Function::Create(FTy, Function::PrivateLinkage, "Pbar", M.get());
1097   Function::Create(FTy, Function::WeakODRLinkage, "Wfoo", M.get());
1098   Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());
1099   Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());
1100 
1101   InstrProfSymtab ProfSymtab;
1102   EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
1103 
1104   StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",
1105                        "Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};
1106 
1107   for (unsigned I = 0; I < sizeof(Funcs) / sizeof(*Funcs); I++) {
1108     Function *F = M->getFunction(Funcs[I]);
1109     ASSERT_TRUE(F != nullptr);
1110     std::string PGOName = getPGOFuncName(*F);
1111     uint64_t Key = IndexedInstrProf::ComputeHash(PGOName);
1112     ASSERT_EQ(StringRef(PGOName),
1113               ProfSymtab.getFuncName(Key));
1114     ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key));
1115   }
1116 }
1117 
1118 // Testing symtab serialization and creator/deserialization interface
1119 // used by coverage map reader, and raw profile reader.
1120 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_compression_test) {
1121   std::vector<std::string> FuncNames1;
1122   std::vector<std::string> FuncNames2;
1123   for (int I = 0; I < 3; I++) {
1124     std::string str;
1125     raw_string_ostream OS(str);
1126     OS << "func_" << I;
1127     FuncNames1.push_back(OS.str());
1128     str.clear();
1129     OS << "f oooooooooooooo_" << I;
1130     FuncNames1.push_back(OS.str());
1131     str.clear();
1132     OS << "BAR_" << I;
1133     FuncNames2.push_back(OS.str());
1134     str.clear();
1135     OS << "BlahblahBlahblahBar_" << I;
1136     FuncNames2.push_back(OS.str());
1137   }
1138 
1139   for (bool DoCompression : {false, true}) {
1140     // Compressing:
1141     std::string FuncNameStrings1;
1142     EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1143                           FuncNames1, (DoCompression && zlib::isAvailable()),
1144                           FuncNameStrings1),
1145                       Succeeded());
1146 
1147     // Compressing:
1148     std::string FuncNameStrings2;
1149     EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1150                           FuncNames2, (DoCompression && zlib::isAvailable()),
1151                           FuncNameStrings2),
1152                       Succeeded());
1153 
1154     for (int Padding = 0; Padding < 2; Padding++) {
1155       // Join with paddings :
1156       std::string FuncNameStrings = FuncNameStrings1;
1157       for (int P = 0; P < Padding; P++) {
1158         FuncNameStrings.push_back('\0');
1159       }
1160       FuncNameStrings += FuncNameStrings2;
1161 
1162       // Now decompress:
1163       InstrProfSymtab Symtab;
1164       EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());
1165 
1166       // Now do the checks:
1167       // First sampling some data points:
1168       StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[0]));
1169       ASSERT_EQ(StringRef("func_0"), R);
1170       R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[1]));
1171       ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);
1172       for (int I = 0; I < 3; I++) {
1173         std::string N[4];
1174         N[0] = FuncNames1[2 * I];
1175         N[1] = FuncNames1[2 * I + 1];
1176         N[2] = FuncNames2[2 * I];
1177         N[3] = FuncNames2[2 * I + 1];
1178         for (int J = 0; J < 4; J++) {
1179           StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(N[J]));
1180           ASSERT_EQ(StringRef(N[J]), R);
1181         }
1182       }
1183     }
1184   }
1185 }
1186 
1187 TEST_P(MaybeSparseInstrProfTest, remapping_test) {
1188   Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);
1189   Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err);
1190   auto Profile = Writer.writeBuffer();
1191   readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(
1192     type i l
1193     name 3bar 4quux
1194   )"));
1195 
1196   std::vector<uint64_t> Counts;
1197   for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1198     EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1199                       Succeeded());
1200     ASSERT_EQ(4u, Counts.size());
1201     EXPECT_EQ(1u, Counts[0]);
1202     EXPECT_EQ(2u, Counts[1]);
1203     EXPECT_EQ(3u, Counts[2]);
1204     EXPECT_EQ(4u, Counts[3]);
1205   }
1206 
1207   for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) {
1208     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1209                       Succeeded());
1210     ASSERT_EQ(3u, Counts.size());
1211     EXPECT_EQ(5u, Counts[0]);
1212     EXPECT_EQ(6u, Counts[1]);
1213     EXPECT_EQ(7u, Counts[2]);
1214   }
1215 
1216   for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1217                             "_Z3barf", "otherfile:_Z4quuxf"}) {
1218     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1219                       Failed());
1220     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1221                       Failed());
1222   }
1223 }
1224 
1225 TEST_F(SparseInstrProfTest, preserve_no_records) {
1226   Writer.addRecord({"foo", 0x1234, {0}}, Err);
1227   Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);
1228   Writer.addRecord({"baz", 0x4321, {0, 0, 0}}, Err);
1229 
1230   auto Profile = Writer.writeBuffer();
1231   readProfile(std::move(Profile));
1232 
1233   auto I = Reader->begin(), E = Reader->end();
1234   ASSERT_TRUE(I == E);
1235 }
1236 
1237 INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,
1238                          ::testing::Bool());
1239 
1240 #if defined(_LP64) && defined(EXPENSIVE_CHECKS)
1241 TEST(ProfileReaderTest, ReadsLargeFiles) {
1242   const size_t LargeSize = 1ULL << 32; // 4GB
1243 
1244   auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1245   if (!RawProfile)
1246     return;
1247   auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
1248   ASSERT_TRUE(InstrProfError::take(RawProfileReaderOrErr.takeError()) ==
1249               instrprof_error::unrecognized_format);
1250 
1251   auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1252   if (!IndexedProfile)
1253     return;
1254   auto IndexedReaderOrErr =
1255       IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
1256   ASSERT_TRUE(InstrProfError::take(IndexedReaderOrErr.takeError()) ==
1257               instrprof_error::bad_magic);
1258 }
1259 #endif
1260 
1261 } // end anonymous namespace
1262