libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp

   1 //===-- Application to analyze benchmark JSON files -----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "automemcpy/ResultAnalyzer.h"
  10 #include "llvm/ADT/StringMap.h"
  11 #include "llvm/ADT/StringSet.h"
  12 #include "llvm/Support/CommandLine.h"
  13 #include "llvm/Support/Error.h"
  14 #include "llvm/Support/JSON.h"
  15 #include "llvm/Support/MemoryBuffer.h"
  16
  17 namespace llvm {
  18
  19 // User can specify one or more json filenames to process on the command line.
  20 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
  21                                             cl::desc("<input json files>"));
  22
  23 // User can filter the distributions to be taken into account.
  24 static cl::list<std::string>
  25     KeepOnlyDistributions("keep-only-distributions",
  26                           cl::desc("<comma separated list of distribution "
  27                                    "names, keeps all if unspecified>"));
  28
  29 namespace automemcpy {
  30
  31 // This is defined in the autogenerated 'Implementations.cpp' file.
  32 extern ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors();
  33
  34 // Iterates over all functions and fills a map of function name to function
  35 // descriptor pointers.
  36 static StringMap<const FunctionDescriptor *> createFunctionDescriptorMap() {
  37   StringMap<const FunctionDescriptor *> Descriptors;
  38   for (const NamedFunctionDescriptor &FD : getFunctionDescriptors())
  39     Descriptors.insert_or_assign(FD.Name, &FD.Desc);
  40   return Descriptors;
  41 }
  42
  43 // Retrieves the function descriptor for a particular function name.
  44 static const FunctionDescriptor &getFunctionDescriptor(StringRef FunctionName) {
  45   static StringMap<const FunctionDescriptor *> Descriptors =
  46       createFunctionDescriptorMap();
  47   const auto *FD = Descriptors.lookup(FunctionName);
  48   if (!FD)
  49     report_fatal_error(
  50         Twine("No FunctionDescriptor for ").concat(FunctionName));
  51   return *FD;
  52 }
  53
  54 // Functions and distributions names are stored quite a few times so it's more
  55 // efficient to internalize these strings and refer to them through 'StringRef'.
  56 static StringRef getInternalizedString(StringRef VolatileStr) {
  57   static llvm::StringSet StringCache;
  58   return StringCache.insert(VolatileStr).first->getKey();
  59 }
  60
  61 // Helper function for the LLVM JSON API.
  62 bool fromJSON(const json::Value &V, Sample &Out, json::Path P) {
  63   std::string Label;
  64   std::string RunType;
  65   json::ObjectMapper O(V, P);
  66   if (O && O.map("bytes_per_second", Out.BytesPerSecond) &&
  67       O.map("run_type", RunType) && O.map("label", Label)) {
  68     const auto LabelPair = StringRef(Label).split(',');
  69     Out.Id.Function.Name = getInternalizedString(LabelPair.first);
  70     Out.Id.Function.Type = getFunctionDescriptor(LabelPair.first).Type;
  71     Out.Id.Distribution.Name = getInternalizedString(LabelPair.second);
  72     Out.Type = StringSwitch<SampleType>(RunType)
  73                    .Case("aggregate", SampleType::AGGREGATE)
  74                    .Case("iteration", SampleType::ITERATION);
  75     return true;
  76   }
  77   return false;
  78 }
  79
  80 // An object to represent the content of the JSON file.
  81 // This is easier to parse/serialize JSON when the structures of the json file
  82 // maps the structure of the object.
  83 struct JsonFile {
  84   std::vector<Sample> Samples;
  85 };
  86
  87 // Helper function for the LLVM JSON API.
  88 bool fromJSON(const json::Value &V, JsonFile &JF, json::Path P) {
  89   json::ObjectMapper O(V, P);
  90   return O && O.map("benchmarks", JF.Samples);
  91 }
  92
  93 // Global object to ease error reporting, it consumes errors and crash the
  94 // application with a meaningful message.
  95 static ExitOnError ExitOnErr;
  96
  97 // Main JSON parsing method. Reads the content of the file pointed to by
  98 // 'Filename' and returns a JsonFile object.
  99 JsonFile parseJsonResultFile(StringRef Filename) {
 100   auto Buf = ExitOnErr(errorOrToExpected(
 101       MemoryBuffer::getFile(Filename, /*bool IsText=*/true,
 102                             /*RequiresNullTerminator=*/false)));
 103   auto JsonValue = ExitOnErr(json::parse(Buf->getBuffer()));
 104   json::Path::Root Root;
 105   JsonFile JF;
 106   if (!fromJSON(JsonValue, JF, Root))
 107     ExitOnErr(Root.getError());
 108   return JF;
 109 }
 110
 111 // Serializes the 'GradeHisto' to the provided 'Stream'.
 112 static void Serialize(raw_ostream &Stream, const GradeHistogram &GH) {
 113   static constexpr std::array<StringRef, 9> kCharacters = {
 114       " ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
 115
 116   const size_t Max = *std::max_element(GH.begin(), GH.end());
 117   for (size_t I = 0; I < GH.size(); ++I) {
 118     size_t Index = (float(GH[I]) / Max) * (kCharacters.size() - 1);
 119     Stream << kCharacters.at(Index);
 120   }
 121 }
 122
 123 int Main(int argc, char **argv) {
 124   ExitOnErr.setBanner("Automemcpy Json Results Analyzer stopped with error: ");
 125   cl::ParseCommandLineOptions(argc, argv, "Automemcpy Json Results Analyzer\n");
 126
 127   // Reads all samples stored in the input JSON files.
 128   std::vector<Sample> Samples;
 129   for (const auto &Filename : InputFilenames) {
 130     auto Result = parseJsonResultFile(Filename);
 131     llvm::append_range(Samples, Result.Samples);
 132   }
 133
 134   if (!KeepOnlyDistributions.empty()) {
 135     llvm::StringSet ValidDistributions;
 136     ValidDistributions.insert(KeepOnlyDistributions.begin(),
 137                               KeepOnlyDistributions.end());
 138     llvm::erase_if(Samples, [&ValidDistributions](const Sample &S) {
 139       return !ValidDistributions.contains(S.Id.Distribution.Name);
 140     });
 141   }
 142
 143   // Extracts median of throughputs.
 144   std::vector<FunctionData> Functions = getThroughputs(Samples);
 145   fillScores(Functions);
 146   castVotes(Functions);
 147
 148   // Present data by function type, Grade and Geomean of scores.
 149   std::sort(Functions.begin(), Functions.end(),
 150             [](const FunctionData &A, const FunctionData &B) {
 151               const auto Less = [](const FunctionData &FD) {
 152                 return std::make_tuple(FD.Id.Type, FD.FinalGrade,
 153                                        -FD.ScoresGeoMean);
 154               };
 155               return Less(A) < Less(B);
 156             });
 157
 158   // Print result.
 159   for (const FunctionData &Function : Functions) {
 160     outs() << formatv("{0,-10}", Grade::getString(Function.FinalGrade));
 161     outs() << " |";
 162     Serialize(outs(), Function.GradeHisto);
 163     outs() << "| ";
 164     outs().resetColor();
 165     outs() << formatv("{0,+25}", Function.Id.Name);
 166     outs() << "\n";
 167   }
 168
 169   return EXIT_SUCCESS;
 170 }
 171
 172 } // namespace automemcpy
 173 } // namespace llvm
 174
 175 int main(int argc, char **argv) { return llvm::automemcpy::Main(argc, argv); }