libc/benchmarks/LibcMemoryBenchmarkMain.cpp

   1 //===-- Benchmark ---------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "JSON.h"
  10 #include "LibcBenchmark.h"
  11 #include "LibcMemoryBenchmark.h"
  12 #include "MemorySizeDistributions.h"
  13 #include "src/__support/macros/config.h"
  14 #include "llvm/Support/CommandLine.h"
  15 #include "llvm/Support/ErrorHandling.h"
  16 #include "llvm/Support/FileSystem.h"
  17 #include "llvm/Support/JSON.h"
  18 #include "llvm/Support/MathExtras.h"
  19 #include "llvm/Support/MemoryBuffer.h"
  20 #include "llvm/Support/raw_ostream.h"
  21
  22 #include <cstring>
  23 #include <unistd.h>
  24
  25 namespace LIBC_NAMESPACE_DECL {
  26
  27 extern void *memcpy(void *__restrict, const void *__restrict, size_t);
  28 extern void *memmove(void *, const void *, size_t);
  29 extern void *memset(void *, int, size_t);
  30 extern void bzero(void *, size_t);
  31 extern int memcmp(const void *, const void *, size_t);
  32 extern int bcmp(const void *, const void *, size_t);
  33
  34 } // namespace LIBC_NAMESPACE_DECL
  35
  36 namespace llvm {
  37 namespace libc_benchmarks {
  38
  39 static cl::opt<std::string>
  40     StudyName("study-name", cl::desc("The name for this study"), cl::Required);
  41
  42 static cl::opt<std::string>
  43     SizeDistributionName("size-distribution-name",
  44                          cl::desc("The name of the distribution to use"));
  45
  46 static cl::opt<bool> SweepMode(
  47     "sweep-mode",
  48     cl::desc(
  49         "If set, benchmark all sizes from sweep-min-size to sweep-max-size"));
  50
  51 static cl::opt<uint32_t>
  52     SweepMinSize("sweep-min-size",
  53                  cl::desc("The minimum size to use in sweep-mode"),
  54                  cl::init(0));
  55
  56 static cl::opt<uint32_t>
  57     SweepMaxSize("sweep-max-size",
  58                  cl::desc("The maximum size to use in sweep-mode"),
  59                  cl::init(256));
  60
  61 static cl::opt<uint32_t>
  62     AlignedAccess("aligned-access",
  63                   cl::desc("The alignment to use when accessing the buffers\n"
  64                            "Default is unaligned\n"
  65                            "Use 0 to disable address randomization"),
  66                   cl::init(1));
  67
  68 static cl::opt<std::string> Output("output",
  69                                    cl::desc("Specify output filename"),
  70                                    cl::value_desc("filename"), cl::init("-"));
  71
  72 static cl::opt<uint32_t>
  73     NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
  74               cl::init(1));
  75
  76 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
  77 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
  78 using BenchmarkSetup = CopySetup;
  79 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE)
  80 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE
  81 using BenchmarkSetup = MoveSetup;
  82 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
  83 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
  84 using BenchmarkSetup = SetSetup;
  85 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
  86 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
  87 using BenchmarkSetup = SetSetup;
  88 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
  89 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
  90 using BenchmarkSetup = ComparisonSetup;
  91 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
  92 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
  93 using BenchmarkSetup = ComparisonSetup;
  94 #else
  95 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
  96 #endif
  97
  98 struct MemfunctionBenchmarkBase : public BenchmarkSetup {
  99   MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {}
 100   virtual ~MemfunctionBenchmarkBase() {}
 101
 102   virtual Study run() = 0;
 103
 104   CircularArrayRef<ParameterBatch::ParameterType>
 105   generateBatch(size_t Iterations) {
 106     randomize();
 107     return cycle(ArrayRef(Parameters), Iterations);
 108   }
 109
 110 protected:
 111   Study createStudy() {
 112     Study Study;
 113     // Setup study.
 114     Study.StudyName = StudyName;
 115     Runtime &RI = Study.Runtime;
 116     RI.Host = HostState::get();
 117     RI.BufferSize = BufferSize;
 118     RI.BatchParameterCount = BatchSize;
 119
 120     BenchmarkOptions &BO = RI.BenchmarkOptions;
 121     BO.MinDuration = std::chrono::milliseconds(1);
 122     BO.MaxDuration = std::chrono::seconds(1);
 123     BO.MaxIterations = 10'000'000U;
 124     BO.MinSamples = 4;
 125     BO.MaxSamples = 1000;
 126     BO.Epsilon = 0.01; // 1%
 127     BO.ScalingFactor = 1.4;
 128
 129     StudyConfiguration &SC = Study.Configuration;
 130     SC.NumTrials = NumTrials;
 131     SC.IsSweepMode = SweepMode;
 132     SC.AccessAlignment = MaybeAlign(AlignedAccess);
 133     SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
 134     return Study;
 135   }
 136
 137   void runTrials(const BenchmarkOptions &Options,
 138                  std::vector<Duration> &Measurements) {
 139     for (size_t i = 0; i < NumTrials; ++i) {
 140       const BenchmarkResult Result = benchmark(
 141           Options, *this, [this](ParameterBatch::ParameterType Parameter) {
 142             return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
 143           });
 144       Measurements.push_back(Result.BestGuess);
 145       reportProgress(Measurements);
 146     }
 147   }
 148
 149   virtual void randomize() = 0;
 150
 151 private:
 152   bool ReportProgress;
 153
 154   void reportProgress(const std::vector<Duration> &Measurements) {
 155     if (!ReportProgress)
 156       return;
 157     static size_t LastPercent = -1;
 158     const size_t TotalSteps = Measurements.capacity();
 159     const size_t Steps = Measurements.size();
 160     const size_t Percent = 100 * Steps / TotalSteps;
 161     if (Percent == LastPercent)
 162       return;
 163     LastPercent = Percent;
 164     size_t I = 0;
 165     errs() << '[';
 166     for (; I <= Percent; ++I)
 167       errs() << '#';
 168     for (; I <= 100; ++I)
 169       errs() << '_';
 170     errs() << "] " << Percent << '%' << '\r';
 171   }
 172 };
 173
 174 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
 175   MemfunctionBenchmarkSweep()
 176       : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
 177                       MaybeAlign(AlignedAccess)) {}
 178
 179   virtual void randomize() override {
 180     for (auto &P : Parameters) {
 181       P.OffsetBytes = OffsetSampler(Gen);
 182       P.SizeBytes = CurrentSweepSize;
 183       checkValid(P);
 184     }
 185   }
 186
 187   virtual Study run() override {
 188     Study Study = createStudy();
 189     Study.Configuration.SweepModeMaxSize = SweepMaxSize;
 190     BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
 191     BO.MinDuration = std::chrono::milliseconds(1);
 192     BO.InitialIterations = 100;
 193     auto &Measurements = Study.Measurements;
 194     Measurements.reserve(NumTrials * SweepMaxSize);
 195     for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) {
 196       CurrentSweepSize = Size;
 197       runTrials(BO, Measurements);
 198     }
 199     return Study;
 200   }
 201
 202 private:
 203   size_t CurrentSweepSize = 0;
 204   OffsetDistribution OffsetSampler;
 205   std::mt19937_64 Gen;
 206 };
 207
 208 struct MemfunctionBenchmarkDistribution final
 209     : public MemfunctionBenchmarkBase {
 210   MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
 211       : Distribution(Distribution), Probabilities(Distribution.Probabilities),
 212         SizeSampler(Probabilities.begin(), Probabilities.end()),
 213         OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
 214                       Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
 215
 216   virtual void randomize() override {
 217     for (auto &P : Parameters) {
 218       P.OffsetBytes = OffsetSampler(Gen);
 219       P.SizeBytes = SizeSampler(Gen);
 220       checkValid(P);
 221     }
 222   }
 223
 224   virtual Study run() override {
 225     Study Study = createStudy();
 226     Study.Configuration.SizeDistributionName = Distribution.Name.str();
 227     BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
 228     BO.MinDuration = std::chrono::milliseconds(10);
 229     BO.InitialIterations = BatchSize * 10;
 230     auto &Measurements = Study.Measurements;
 231     Measurements.reserve(NumTrials);
 232     runTrials(BO, Measurements);
 233     return Study;
 234   }
 235
 236 private:
 237   MemorySizeDistribution Distribution;
 238   ArrayRef<double> Probabilities;
 239   std::discrete_distribution<unsigned> SizeSampler;
 240   OffsetDistribution OffsetSampler;
 241   std::mt19937_64 Gen;
 242 };
 243
 244 void writeStudy(const Study &S) {
 245   std::error_code EC;
 246   raw_fd_ostream FOS(Output, EC);
 247   if (EC)
 248     report_fatal_error(Twine("Could not open file: ")
 249                            .concat(EC.message())
 250                            .concat(", ")
 251                            .concat(Output));
 252   json::OStream JOS(FOS);
 253   serializeToJson(S, JOS);
 254   FOS << "\n";
 255 }
 256
 257 void main() {
 258   checkRequirements();
 259   if (!isPowerOf2_32(AlignedAccess))
 260     report_fatal_error(AlignedAccess.ArgStr +
 261                        Twine(" must be a power of two or zero"));
 262
 263   const bool HasDistributionName = !SizeDistributionName.empty();
 264   if (SweepMode && HasDistributionName)
 265     report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) +
 266                        "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
 267
 268   std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
 269   if (SweepMode)
 270     Benchmark.reset(new MemfunctionBenchmarkSweep());
 271   else
 272     Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
 273         BenchmarkSetup::getDistributions(), SizeDistributionName)));
 274   writeStudy(Benchmark->run());
 275 }
 276
 277 } // namespace libc_benchmarks
 278 } // namespace llvm
 279
 280 #ifndef NDEBUG
 281 #error For reproducibility benchmarks should not be compiled in DEBUG mode.
 282 #endif
 283
 284 int main(int argc, char **argv) {
 285   llvm::cl::ParseCommandLineOptions(argc, argv);
 286   llvm::libc_benchmarks::main();
 287   return EXIT_SUCCESS;
 288 }