1 //===-- Benchmark ---------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "JSON.h"
10 #include "LibcBenchmark.h"
11 #include "LibcMemoryBenchmark.h"
12 #include "MemorySizeDistributions.h"
13 #include "llvm/Support/CommandLine.h"
14 #include "llvm/Support/ErrorHandling.h"
15 #include "llvm/Support/FileSystem.h"
16 #include "llvm/Support/JSON.h"
17 #include "llvm/Support/MathExtras.h"
18 #include "llvm/Support/MemoryBuffer.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #include <cstring>
22 #include <unistd.h>
23 
24 namespace __llvm_libc {
25 
26 extern void *memcpy(void *__restrict, const void *__restrict, size_t);
27 extern void *memset(void *, int, size_t);
28 extern void bzero(void *, size_t);
29 extern int memcmp(const void *, const void *, size_t);
30 extern int bcmp(const void *, const void *, size_t);
31 
32 } // namespace __llvm_libc
33 
34 namespace llvm {
35 namespace libc_benchmarks {
36 
37 static cl::opt<std::string>
38     StudyName("study-name", cl::desc("The name for this study"), cl::Required);
39 
40 static cl::opt<std::string>
41     SizeDistributionName("size-distribution-name",
42                          cl::desc("The name of the distribution to use"));
43 
44 static cl::opt<bool>
45     SweepMode("sweep-mode",
46               cl::desc("If set, benchmark all sizes from 0 to sweep-max-size"));
47 
48 static cl::opt<uint32_t>
49     SweepMaxSize("sweep-max-size",
50                  cl::desc("The maximum size to use in sweep-mode"),
51                  cl::init(256));
52 
53 static cl::opt<uint32_t>
54     AlignedAccess("aligned-access",
55                   cl::desc("The alignment to use when accessing the buffers\n"
56                            "Default is unaligned\n"
57                            "Use 0 to disable address randomization"),
58                   cl::init(1));
59 
60 static cl::opt<std::string> Output("output",
61                                    cl::desc("Specify output filename"),
62                                    cl::value_desc("filename"), cl::init("-"));
63 
64 static cl::opt<uint32_t>
65     NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
66               cl::init(1));
67 
68 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
69 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
70 using BenchmarkSetup = CopySetup;
71 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
72 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
73 using BenchmarkSetup = SetSetup;
74 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
75 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
76 using BenchmarkSetup = SetSetup;
77 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
78 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
79 using BenchmarkSetup = ComparisonSetup;
80 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
81 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
82 using BenchmarkSetup = ComparisonSetup;
83 #else
84 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
85 #endif
86 
87 struct MemfunctionBenchmarkBase : public BenchmarkSetup {
88   MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {}
89   virtual ~MemfunctionBenchmarkBase() {}
90 
91   virtual Study run() = 0;
92 
93   CircularArrayRef<ParameterBatch::ParameterType>
94   generateBatch(size_t Iterations) {
95     randomize();
96     return cycle(makeArrayRef(Parameters), Iterations);
97   }
98 
99 protected:
100   Study createStudy() {
101     Study Study;
102     // Setup study.
103     Study.StudyName = StudyName;
104     Runtime &RI = Study.Runtime;
105     RI.Host = HostState::get();
106     RI.BufferSize = BufferSize;
107     RI.BatchParameterCount = BatchSize;
108 
109     BenchmarkOptions &BO = RI.BenchmarkOptions;
110     BO.MinDuration = std::chrono::milliseconds(1);
111     BO.MaxDuration = std::chrono::seconds(1);
112     BO.MaxIterations = 10'000'000U;
113     BO.MinSamples = 4;
114     BO.MaxSamples = 1000;
115     BO.Epsilon = 0.01; // 1%
116     BO.ScalingFactor = 1.4;
117 
118     StudyConfiguration &SC = Study.Configuration;
119     SC.NumTrials = NumTrials;
120     SC.IsSweepMode = SweepMode;
121     SC.AccessAlignment = MaybeAlign(AlignedAccess);
122     SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
123     return Study;
124   }
125 
126   void runTrials(const BenchmarkOptions &Options,
127                  std::vector<Duration> &Measurements) {
128     for (size_t i = 0; i < NumTrials; ++i) {
129       const BenchmarkResult Result = benchmark(
130           Options, *this, [this](ParameterBatch::ParameterType Parameter) {
131             return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
132           });
133       Measurements.push_back(Result.BestGuess);
134       reportProgress(Measurements);
135     }
136   }
137 
138   virtual void randomize() = 0;
139 
140 private:
141   bool ReportProgress;
142 
143   void reportProgress(const std::vector<Duration> &Measurements) {
144     if (!ReportProgress)
145       return;
146     static size_t LastPercent = -1;
147     const size_t TotalSteps = Measurements.capacity();
148     const size_t Steps = Measurements.size();
149     const size_t Percent = 100 * Steps / TotalSteps;
150     if (Percent == LastPercent)
151       return;
152     LastPercent = Percent;
153     size_t I = 0;
154     errs() << '[';
155     for (; I <= Percent; ++I)
156       errs() << '#';
157     for (; I <= 100; ++I)
158       errs() << '_';
159     errs() << "] " << Percent << '%' << '\r';
160   }
161 };
162 
163 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
164   MemfunctionBenchmarkSweep()
165       : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
166                       MaybeAlign(AlignedAccess)) {}
167 
168   virtual void randomize() override {
169     for (auto &P : Parameters) {
170       P.OffsetBytes = OffsetSampler(Gen);
171       P.SizeBytes = CurrentSweepSize;
172       checkValid(P);
173     }
174   }
175 
176   virtual Study run() override {
177     Study Study = createStudy();
178     Study.Configuration.SweepModeMaxSize = SweepMaxSize;
179     BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
180     BO.MinDuration = std::chrono::milliseconds(1);
181     BO.InitialIterations = 100;
182     auto &Measurements = Study.Measurements;
183     Measurements.reserve(NumTrials * SweepMaxSize);
184     for (size_t Size = 0; Size <= SweepMaxSize; ++Size) {
185       CurrentSweepSize = Size;
186       runTrials(BO, Measurements);
187     }
188     return Study;
189   }
190 
191 private:
192   size_t CurrentSweepSize = 0;
193   OffsetDistribution OffsetSampler;
194   std::mt19937_64 Gen;
195 };
196 
197 struct MemfunctionBenchmarkDistribution final
198     : public MemfunctionBenchmarkBase {
199   MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
200       : Distribution(Distribution), Probabilities(Distribution.Probabilities),
201         SizeSampler(Probabilities.begin(), Probabilities.end()),
202         OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
203                       Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
204 
205   virtual void randomize() override {
206     for (auto &P : Parameters) {
207       P.OffsetBytes = OffsetSampler(Gen);
208       P.SizeBytes = SizeSampler(Gen);
209       checkValid(P);
210     }
211   }
212 
213   virtual Study run() override {
214     Study Study = createStudy();
215     Study.Configuration.SizeDistributionName = Distribution.Name.str();
216     BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
217     BO.MinDuration = std::chrono::milliseconds(10);
218     BO.InitialIterations = BatchSize * 10;
219     auto &Measurements = Study.Measurements;
220     Measurements.reserve(NumTrials);
221     runTrials(BO, Measurements);
222     return Study;
223   }
224 
225 private:
226   MemorySizeDistribution Distribution;
227   ArrayRef<double> Probabilities;
228   std::discrete_distribution<unsigned> SizeSampler;
229   OffsetDistribution OffsetSampler;
230   std::mt19937_64 Gen;
231 };
232 
233 void writeStudy(const Study &S) {
234   std::error_code EC;
235   raw_fd_ostream FOS(Output, EC);
236   if (EC)
237     report_fatal_error(Twine("Could not open file: ")
238                            .concat(EC.message())
239                            .concat(", ")
240                            .concat(Output));
241   json::OStream JOS(FOS);
242   serializeToJson(S, JOS);
243   FOS << "\n";
244 }
245 
246 void main() {
247   checkRequirements();
248   if (!isPowerOf2_32(AlignedAccess))
249     report_fatal_error(AlignedAccess.ArgStr +
250                        Twine(" must be a power of two or zero"));
251 
252   const bool HasDistributionName = !SizeDistributionName.empty();
253   if (SweepMode && HasDistributionName)
254     report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) +
255                        "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
256 
257   std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
258   if (SweepMode)
259     Benchmark.reset(new MemfunctionBenchmarkSweep());
260   else
261     Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
262         BenchmarkSetup::getDistributions(), SizeDistributionName)));
263   writeStudy(Benchmark->run());
264 }
265 
266 } // namespace libc_benchmarks
267 } // namespace llvm
268 
269 #ifndef NDEBUG
270 #error For reproducibility benchmarks should not be compiled in DEBUG mode.
271 #endif
272 
273 int main(int argc, char **argv) {
274   llvm::cl::ParseCommandLineOptions(argc, argv);
275   llvm::libc_benchmarks::main();
276   return EXIT_SUCCESS;
277 }
278