1 //===-- Analyze benchmark JSON files ----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
10 #define LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
11 
12 #include "automemcpy/FunctionDescriptor.h"
13 #include "llvm/ADT/ArrayRef.h"
14 #include "llvm/ADT/StringMap.h"
15 #include <array>
16 #include <vector>
17 
18 namespace llvm {
19 namespace automemcpy {
20 
21 // A Grade as in the Majority Judgment voting system.
22 struct Grade {
23   enum GradeEnum {
24     EXCELLENT,
25     VERY_GOOD,
26     GOOD,
27     PASSABLE,
28     INADEQUATE,
29     MEDIOCRE,
30     BAD,
31     ARRAY_SIZE,
32   };
33 
34   // Returns a human readable string of the enum.
35   static StringRef getString(const GradeEnum &GE);
36 
37   // Turns 'Score' into a GradeEnum.
38   static GradeEnum judge(double Score);
39 };
40 
41 // A 'GradeEnum' indexed array with counts for each grade.
42 using GradeHistogram = std::array<size_t, Grade::ARRAY_SIZE>;
43 
44 // Identifies a Function by its name and type. Used as a key in a map.
45 struct FunctionId {
46   StringRef Name;
47   FunctionType Type;
48   COMPARABLE_AND_HASHABLE(FunctionId, Type, Name)
49 };
50 
51 struct PerDistributionData {
52   std::vector<double> BytesPerSecondSamples;
53   double BytesPerSecondMedian;   // Median of samples for this distribution.
54   double BytesPerSecondMean;     // Mean of samples for this distribution.
55   double BytesPerSecondVariance; // Variance of samples for this distribution.
56   double Score;                  // Normalized score for this distribution.
57   Grade::GradeEnum Grade;        // Grade for this distribution.
58 };
59 
60 struct FunctionData {
61   FunctionId Id;
62   StringMap<PerDistributionData> PerDistributionData;
63   double ScoresGeoMean;           // Geomean of scores for each distribution.
64   GradeHistogram GradeHisto = {}; // GradeEnum indexed array
65   Grade::GradeEnum FinalGrade = Grade::BAD; // Overall grade for this function
66 };
67 
68 // Identifies a Distribution by its name. Used as a key in a map.
69 struct DistributionId {
70   StringRef Name;
71   COMPARABLE_AND_HASHABLE(DistributionId, Name)
72 };
73 
74 // Identifies a Sample by its distribution and function. Used as a key in a map.
75 struct SampleId {
76   FunctionId Function;
77   DistributionId Distribution;
78   COMPARABLE_AND_HASHABLE(SampleId, Function.Type, Function.Name,
79                           Distribution.Name)
80 };
81 
82 // The type of Samples as reported by the Google Benchmark's JSON result file.
83 // We are only interested in the "iteration" samples, the "aggregate" ones
84 // represent derived metrics such as 'mean' or 'median'.
85 enum class SampleType { UNKNOWN, ITERATION, AGGREGATE };
86 
87 // A SampleId with an associated measured throughput.
88 struct Sample {
89   SampleId Id;
90   SampleType Type = SampleType::UNKNOWN;
91   double BytesPerSecond = 0;
92 };
93 
94 // This function collects Samples that belong to the same distribution and
95 // function and retains the median one. It then stores each of them into a
96 // 'FunctionData' and returns them as a vector.
97 std::vector<FunctionData> getThroughputs(ArrayRef<Sample> Samples);
98 
99 // Normalize the function's throughput per distribution.
100 void fillScores(MutableArrayRef<FunctionData> Functions);
101 
102 // Convert scores into Grades, stores an histogram of Grade for each functions
103 // and cast a median grade for the function.
104 void castVotes(MutableArrayRef<FunctionData> Functions);
105 
106 } // namespace automemcpy
107 } // namespace llvm
108 
109 #endif // LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
110