1 //===-- Automemcpy Json Results Analyzer Test ----------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "automemcpy/ResultAnalyzer.h" 10 #include "gmock/gmock.h" 11 #include "gtest/gtest.h" 12 13 using testing::DoubleNear; 14 using testing::ElementsAre; 15 using testing::Pair; 16 using testing::SizeIs; 17 18 namespace llvm { 19 namespace automemcpy { 20 namespace { 21 22 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsOneSample) { 23 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY}; 24 static constexpr DistributionId DistA = {{"A"}}; 25 static constexpr SampleId Id = {Foo1, DistA}; 26 static constexpr Sample kSamples[] = { 27 Sample{Id, 4}, 28 }; 29 30 const std::vector<FunctionData> Data = getThroughputs(kSamples); 31 EXPECT_THAT(Data, SizeIs(1)); 32 EXPECT_THAT(Data[0].Id, Foo1); 33 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1)); 34 // A single value is provided. 35 const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name); 36 EXPECT_THAT(DistributionData.BytesPerSecondMedian, 4); 37 EXPECT_THAT(DistributionData.BytesPerSecondMean, 4); 38 EXPECT_THAT(DistributionData.BytesPerSecondVariance, 0); 39 } 40 41 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsManySamplesSameBucket) { 42 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY}; 43 static constexpr DistributionId DistA = {{"A"}}; 44 static constexpr SampleId Id = {Foo1, DistA}; 45 static constexpr Sample kSamples[] = {Sample{Id, 4}, Sample{Id, 5}, 46 Sample{Id, 5}}; 47 48 const std::vector<FunctionData> Data = getThroughputs(kSamples); 49 EXPECT_THAT(Data, SizeIs(1)); 50 EXPECT_THAT(Data[0].Id, Foo1); 51 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1)); 52 // When multiple values are provided we pick the median one (here median of 4, 53 // 5, 5). 54 const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name); 55 EXPECT_THAT(DistributionData.BytesPerSecondMedian, 5); 56 EXPECT_THAT(DistributionData.BytesPerSecondMean, DoubleNear(4.6, 0.1)); 57 EXPECT_THAT(DistributionData.BytesPerSecondVariance, DoubleNear(0.33, 0.01)); 58 } 59 60 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsServeralFunctionAndDist) { 61 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY}; 62 static constexpr DistributionId DistA = {{"A"}}; 63 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY}; 64 static constexpr DistributionId DistB = {{"B"}}; 65 static constexpr Sample kSamples[] = { 66 Sample{{Foo1, DistA}, 1}, Sample{{Foo1, DistB}, 2}, 67 Sample{{Foo2, DistA}, 3}, Sample{{Foo2, DistB}, 4}}; 68 // Data is aggregated per function. 69 const std::vector<FunctionData> Data = getThroughputs(kSamples); 70 EXPECT_THAT(Data, SizeIs(2)); // 2 functions Foo1 and Foo2. 71 // Each function has data for both distributions DistA and DistB. 72 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(2)); 73 EXPECT_THAT(Data[1].PerDistributionData, SizeIs(2)); 74 } 75 76 TEST(AutomemcpyJsonResultsAnalyzer, getScore) { 77 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY}; 78 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY}; 79 static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY}; 80 static constexpr DistributionId Dist = {{"A"}}; 81 static constexpr Sample kSamples[] = {Sample{{Foo1, Dist}, 1}, 82 Sample{{Foo2, Dist}, 2}, 83 Sample{{Foo3, Dist}, 3}}; 84 85 // Data is aggregated per function. 86 std::vector<FunctionData> Data = getThroughputs(kSamples); 87 88 // Sort Data by function name so we can test them. 89 std::sort( 90 Data.begin(), Data.end(), 91 [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; }); 92 93 EXPECT_THAT(Data[0].Id, Foo1); 94 EXPECT_THAT(Data[0].PerDistributionData.lookup("A").BytesPerSecondMedian, 1); 95 EXPECT_THAT(Data[1].Id, Foo2); 96 EXPECT_THAT(Data[1].PerDistributionData.lookup("A").BytesPerSecondMedian, 2); 97 EXPECT_THAT(Data[2].Id, Foo3); 98 EXPECT_THAT(Data[2].PerDistributionData.lookup("A").BytesPerSecondMedian, 3); 99 100 // Normalizes throughput per distribution. 101 fillScores(Data); 102 EXPECT_THAT(Data[0].PerDistributionData.lookup("A").Score, 0); 103 EXPECT_THAT(Data[1].PerDistributionData.lookup("A").Score, 0.5); 104 EXPECT_THAT(Data[2].PerDistributionData.lookup("A").Score, 1); 105 } 106 107 TEST(AutomemcpyJsonResultsAnalyzer, castVotes) { 108 static constexpr double kAbsErr = 0.01; 109 110 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY}; 111 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY}; 112 static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY}; 113 static constexpr DistributionId DistA = {{"A"}}; 114 static constexpr DistributionId DistB = {{"B"}}; 115 static constexpr Sample kSamples[] = { 116 Sample{{Foo1, DistA}, 0}, Sample{{Foo1, DistB}, 30}, 117 Sample{{Foo2, DistA}, 1}, Sample{{Foo2, DistB}, 100}, 118 Sample{{Foo3, DistA}, 7}, Sample{{Foo3, DistB}, 100}, 119 }; 120 121 // DistA Thoughput ranges from 0 to 7. 122 // DistB Thoughput ranges from 30 to 100. 123 124 // Data is aggregated per function. 125 std::vector<FunctionData> Data = getThroughputs(kSamples); 126 127 // Sort Data by function name so we can test them. 128 std::sort( 129 Data.begin(), Data.end(), 130 [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; }); 131 132 // Normalizes throughput per distribution. 133 fillScores(Data); 134 135 // Cast votes 136 castVotes(Data); 137 138 EXPECT_THAT(Data[0].Id, Foo1); 139 EXPECT_THAT(Data[1].Id, Foo2); 140 EXPECT_THAT(Data[2].Id, Foo3); 141 142 const auto GetDistData = [&Data](size_t Index, StringRef Name) { 143 return Data[Index].PerDistributionData.lookup(Name); 144 }; 145 146 // Distribution A 147 // Throughput is 0, 1 and 7, so normalized scores are 0, 1/7 and 1. 148 EXPECT_THAT(GetDistData(0, "A").Score, DoubleNear(0, kAbsErr)); 149 EXPECT_THAT(GetDistData(1, "A").Score, DoubleNear(1. / 7, kAbsErr)); 150 EXPECT_THAT(GetDistData(2, "A").Score, DoubleNear(1, kAbsErr)); 151 // which are turned into grades BAD, MEDIOCRE and EXCELLENT. 152 EXPECT_THAT(GetDistData(0, "A").Grade, Grade::BAD); 153 EXPECT_THAT(GetDistData(1, "A").Grade, Grade::MEDIOCRE); 154 EXPECT_THAT(GetDistData(2, "A").Grade, Grade::EXCELLENT); 155 156 // Distribution B 157 // Throughput is 30, 100 and 100, so normalized scores are 0, 1 and 1. 158 EXPECT_THAT(GetDistData(0, "B").Score, DoubleNear(0, kAbsErr)); 159 EXPECT_THAT(GetDistData(1, "B").Score, DoubleNear(1, kAbsErr)); 160 EXPECT_THAT(GetDistData(2, "B").Score, DoubleNear(1, kAbsErr)); 161 // which are turned into grades BAD, EXCELLENT and EXCELLENT. 162 EXPECT_THAT(GetDistData(0, "B").Grade, Grade::BAD); 163 EXPECT_THAT(GetDistData(1, "B").Grade, Grade::EXCELLENT); 164 EXPECT_THAT(GetDistData(2, "B").Grade, Grade::EXCELLENT); 165 166 // Now looking from the functions point of view. 167 EXPECT_THAT(Data[0].ScoresGeoMean, DoubleNear(0, kAbsErr)); 168 EXPECT_THAT(Data[1].ScoresGeoMean, DoubleNear(1. * (1. / 7), kAbsErr)); 169 EXPECT_THAT(Data[2].ScoresGeoMean, DoubleNear(1, kAbsErr)); 170 171 // Note the array is indexed by GradeEnum values (EXCELLENT=0 / BAD = 6) 172 EXPECT_THAT(Data[0].GradeHisto, ElementsAre(0, 0, 0, 0, 0, 0, 2)); 173 EXPECT_THAT(Data[1].GradeHisto, ElementsAre(1, 0, 0, 0, 0, 1, 0)); 174 EXPECT_THAT(Data[2].GradeHisto, ElementsAre(2, 0, 0, 0, 0, 0, 0)); 175 176 EXPECT_THAT(Data[0].FinalGrade, Grade::BAD); 177 EXPECT_THAT(Data[1].FinalGrade, Grade::MEDIOCRE); 178 EXPECT_THAT(Data[2].FinalGrade, Grade::EXCELLENT); 179 } 180 181 } // namespace 182 } // namespace automemcpy 183 } // namespace llvm 184