1 //===-- Application to analyze benchmark JSON files -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "automemcpy/ResultAnalyzer.h"
10 #include "llvm/ADT/StringMap.h"
11 #include "llvm/ADT/StringSet.h"
12 #include "llvm/Support/CommandLine.h"
13 #include "llvm/Support/Error.h"
14 #include "llvm/Support/JSON.h"
15 #include "llvm/Support/MemoryBuffer.h"
16 
17 namespace llvm {
18 
19 // User can specify one or more json filenames to process on the command line.
20 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
21                                             cl::desc("<input json files>"));
22 
23 // User can filter the distributions to be taken into account.
24 static cl::list<std::string>
25     KeepOnlyDistributions("keep-only-distributions",
26                           cl::desc("<comma separated list of distribution "
27                                    "names, keeps all if unspecified>"));
28 
29 namespace automemcpy {
30 
31 // This is defined in the autogenerated 'Implementations.cpp' file.
32 extern ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors();
33 
34 // Iterates over all functions and fills a map of function name to function
35 // descriptor pointers.
36 static StringMap<const FunctionDescriptor *> createFunctionDescriptorMap() {
37   StringMap<const FunctionDescriptor *> Descriptors;
38   for (const NamedFunctionDescriptor &FD : getFunctionDescriptors())
39     Descriptors.insert_or_assign(FD.Name, &FD.Desc);
40   return Descriptors;
41 }
42 
43 // Retrieves the function descriptor for a particular function name.
44 static const FunctionDescriptor &getFunctionDescriptor(StringRef FunctionName) {
45   static StringMap<const FunctionDescriptor *> Descriptors =
46       createFunctionDescriptorMap();
47   const auto *FD = Descriptors.lookup(FunctionName);
48   if (!FD)
49     report_fatal_error(
50         Twine("No FunctionDescriptor for ").concat(FunctionName));
51   return *FD;
52 }
53 
54 // Functions and distributions names are stored quite a few times so it's more
55 // efficient to internalize these strings and refer to them through 'StringRef'.
56 static StringRef getInternalizedString(StringRef VolatileStr) {
57   static llvm::StringSet StringCache;
58   return StringCache.insert(VolatileStr).first->getKey();
59 }
60 
61 // Helper function for the LLVM JSON API.
62 bool fromJSON(const json::Value &V, Sample &Out, json::Path P) {
63   std::string Label;
64   std::string RunType;
65   json::ObjectMapper O(V, P);
66   if (O && O.map("bytes_per_second", Out.BytesPerSecond) &&
67       O.map("run_type", RunType) && O.map("label", Label)) {
68     const auto LabelPair = StringRef(Label).split(',');
69     Out.Id.Function.Name = getInternalizedString(LabelPair.first);
70     Out.Id.Function.Type = getFunctionDescriptor(LabelPair.first).Type;
71     Out.Id.Distribution.Name = getInternalizedString(LabelPair.second);
72     Out.Type = StringSwitch<SampleType>(RunType)
73                    .Case("aggregate", SampleType::AGGREGATE)
74                    .Case("iteration", SampleType::ITERATION);
75     return true;
76   }
77   return false;
78 }
79 
80 // An object to represent the content of the JSON file.
81 // This is easier to parse/serialize JSON when the structures of the json file
82 // maps the structure of the object.
83 struct JsonFile {
84   std::vector<Sample> Samples;
85 };
86 
87 // Helper function for the LLVM JSON API.
88 bool fromJSON(const json::Value &V, JsonFile &JF, json::Path P) {
89   json::ObjectMapper O(V, P);
90   return O && O.map("benchmarks", JF.Samples);
91 }
92 
93 // Global object to ease error reporting, it consumes errors and crash the
94 // application with a meaningful message.
95 static ExitOnError ExitOnErr;
96 
97 // Main JSON parsing method. Reads the content of the file pointed to by
98 // 'Filename' and returns a JsonFile object.
99 JsonFile parseJsonResultFile(StringRef Filename) {
100   auto Buf = ExitOnErr(errorOrToExpected(
101       MemoryBuffer::getFile(Filename, /*bool IsText=*/true,
102                             /*RequiresNullTerminator=*/false)));
103   auto JsonValue = ExitOnErr(json::parse(Buf->getBuffer()));
104   json::Path::Root Root;
105   JsonFile JF;
106   if (!fromJSON(JsonValue, JF, Root))
107     ExitOnErr(Root.getError());
108   return JF;
109 }
110 
111 // Serializes the 'GradeHisto' to the provided 'Stream'.
112 static void Serialize(raw_ostream &Stream, const GradeHistogram &GH) {
113   static constexpr std::array<StringRef, 9> kCharacters = {
114       " ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
115 
116   const size_t Max = *std::max_element(GH.begin(), GH.end());
117   for (size_t I = 0; I < GH.size(); ++I) {
118     size_t Index = (float(GH[I]) / Max) * (kCharacters.size() - 1);
119     Stream << kCharacters.at(Index);
120   }
121 }
122 
123 int Main(int argc, char **argv) {
124   ExitOnErr.setBanner("Automemcpy Json Results Analyzer stopped with error: ");
125   cl::ParseCommandLineOptions(argc, argv, "Automemcpy Json Results Analyzer\n");
126 
127   // Reads all samples stored in the input JSON files.
128   std::vector<Sample> Samples;
129   for (const auto &Filename : InputFilenames) {
130     auto Result = parseJsonResultFile(Filename);
131     llvm::append_range(Samples, Result.Samples);
132   }
133 
134   if (!KeepOnlyDistributions.empty()) {
135     llvm::StringSet ValidDistributions;
136     ValidDistributions.insert(KeepOnlyDistributions.begin(),
137                               KeepOnlyDistributions.end());
138     llvm::erase_if(Samples, [&ValidDistributions](const Sample &S) {
139       return !ValidDistributions.contains(S.Id.Distribution.Name);
140     });
141   }
142 
143   // Extracts median of throughputs.
144   std::vector<FunctionData> Functions = getThroughputs(Samples);
145   fillScores(Functions);
146   castVotes(Functions);
147 
148   // Present data by function type, Grade and Geomean of scores.
149   std::sort(Functions.begin(), Functions.end(),
150             [](const FunctionData &A, const FunctionData &B) {
151               const auto Less = [](const FunctionData &FD) {
152                 return std::make_tuple(FD.Id.Type, FD.FinalGrade,
153                                        -FD.ScoresGeoMean);
154               };
155               return Less(A) < Less(B);
156             });
157 
158   // Print result.
159   for (const FunctionData &Function : Functions) {
160     outs() << formatv("{0,-10}", Grade::getString(Function.FinalGrade));
161     outs() << " |";
162     Serialize(outs(), Function.GradeHisto);
163     outs() << "| ";
164     outs().resetColor();
165     outs() << formatv("{0,+25}", Function.Id.Name);
166     outs() << "\n";
167   }
168 
169   return EXIT_SUCCESS;
170 }
171 
172 } // namespace automemcpy
173 } // namespace llvm
174 
175 int main(int argc, char **argv) { return llvm::automemcpy::Main(argc, argv); }
176