tools/llvm-exegesis/lib/BenchmarkResult.cpp

   1 //===-- BenchmarkResult.cpp -------------------------------------*- C++ -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "BenchmarkResult.h"
  11 #include "BenchmarkRunner.h"
  12 #include "llvm/ADT/STLExtras.h"
  13 #include "llvm/ADT/bit.h"
  14 #include "llvm/ADT/StringRef.h"
  15 #include "llvm/ObjectYAML/YAML.h"
  16 #include "llvm/Support/FileOutputBuffer.h"
  17 #include "llvm/Support/FileSystem.h"
  18 #include "llvm/Support/Format.h"
  19 #include "llvm/Support/raw_ostream.h"
  20
  21 static constexpr const char kIntegerPrefix[] = "i_0x";
  22 static constexpr const char kDoublePrefix[] = "f_";
  23 static constexpr const char kInvalidOperand[] = "INVALID";
  24
  25 // A mutable struct holding an LLVMState that can be passed through the
  26 // serialization process to encode/decode registers and instructions.
  27 struct YamlContext {
  28   YamlContext(const exegesis::LLVMState &State)
  29       : State(&State), ErrorStream(LastError) {}
  30
  31   void serializeMCInst(const llvm::MCInst &MCInst, llvm::raw_ostream &OS) {
  32     OS << getInstrName(MCInst.getOpcode());
  33     for (const auto &Op : MCInst) {
  34       OS << ' ';
  35       serializeMCOperand(Op, OS);
  36     }
  37   }
  38
  39   void deserializeMCInst(llvm::StringRef String, llvm::MCInst &Value) {
  40     llvm::SmallVector<llvm::StringRef, 8> Pieces;
  41     String.split(Pieces, " ", /* MaxSplit */ -1, /* KeepEmpty */ false);
  42     if (Pieces.empty()) {
  43       ErrorStream << "Unknown Instruction: '" << String << "'";
  44       return;
  45     }
  46     bool ProcessOpcode = true;
  47     for (llvm::StringRef Piece : Pieces) {
  48       if (ProcessOpcode)
  49         Value.setOpcode(getInstrOpcode(Piece));
  50       else
  51         Value.addOperand(deserializeMCOperand(Piece));
  52       ProcessOpcode = false;
  53     }
  54   }
  55
  56   std::string &getLastError() { return ErrorStream.str(); }
  57
  58   llvm::raw_string_ostream &getErrorStream() { return ErrorStream; }
  59
  60   llvm::StringRef getRegName(unsigned RegNo) {
  61     const llvm::StringRef RegName = State->getRegInfo().getName(RegNo);
  62     if (RegName.empty())
  63       ErrorStream << "No register with enum value" << RegNo;
  64     return RegName;
  65   }
  66
  67   unsigned getRegNo(llvm::StringRef RegName) {
  68     const llvm::MCRegisterInfo &RegInfo = State->getRegInfo();
  69     for (unsigned E = RegInfo.getNumRegs(), I = 0; I < E; ++I)
  70       if (RegInfo.getName(I) == RegName)
  71         return I;
  72     ErrorStream << "No register with name " << RegName;
  73     return 0;
  74   }
  75
  76 private:
  77   void serializeIntegerOperand(llvm::raw_ostream &OS, int64_t Value) {
  78     OS << kIntegerPrefix;
  79     OS.write_hex(llvm::bit_cast<uint64_t>(Value));
  80   }
  81
  82   bool tryDeserializeIntegerOperand(llvm::StringRef String, int64_t &Value) {
  83     if (!String.consume_front(kIntegerPrefix))
  84       return false;
  85     return !String.consumeInteger(16, Value);
  86   }
  87
  88   void serializeFPOperand(llvm::raw_ostream &OS, double Value) {
  89     OS << kDoublePrefix << llvm::format("%la", Value);
  90   }
  91
  92   bool tryDeserializeFPOperand(llvm::StringRef String, double &Value) {
  93     if (!String.consume_front(kDoublePrefix))
  94       return false;
  95     char *EndPointer = nullptr;
  96     Value = strtod(String.begin(), &EndPointer);
  97     return EndPointer == String.end();
  98   }
  99
 100   void serializeMCOperand(const llvm::MCOperand &MCOperand,
 101                           llvm::raw_ostream &OS) {
 102     if (MCOperand.isReg()) {
 103       OS << getRegName(MCOperand.getReg());
 104     } else if (MCOperand.isImm()) {
 105       serializeIntegerOperand(OS, MCOperand.getImm());
 106     } else if (MCOperand.isFPImm()) {
 107       serializeFPOperand(OS, MCOperand.getFPImm());
 108     } else {
 109       OS << kInvalidOperand;
 110     }
 111   }
 112
 113   llvm::MCOperand deserializeMCOperand(llvm::StringRef String) {
 114     assert(!String.empty());
 115     int64_t IntValue = 0;
 116     double DoubleValue = 0;
 117     if (tryDeserializeIntegerOperand(String, IntValue))
 118       return llvm::MCOperand::createImm(IntValue);
 119     if (tryDeserializeFPOperand(String, DoubleValue))
 120       return llvm::MCOperand::createFPImm(DoubleValue);
 121     if (unsigned RegNo = getRegNo(String))
 122       return llvm::MCOperand::createReg(RegNo);
 123     if (String != kInvalidOperand)
 124       ErrorStream << "Unknown Operand: '" << String << "'";
 125     return {};
 126   }
 127
 128   llvm::StringRef getInstrName(unsigned InstrNo) {
 129     const llvm::StringRef InstrName = State->getInstrInfo().getName(InstrNo);
 130     if (InstrName.empty())
 131       ErrorStream << "No opcode with enum value" << InstrNo;
 132     return InstrName;
 133   }
 134
 135   unsigned getInstrOpcode(llvm::StringRef InstrName) {
 136     const llvm::MCInstrInfo &InstrInfo = State->getInstrInfo();
 137     for (unsigned E = InstrInfo.getNumOpcodes(), I = 0; I < E; ++I)
 138       if (InstrInfo.getName(I) == InstrName)
 139         return I;
 140     ErrorStream << "No opcode with name " << InstrName;
 141     return 0;
 142   }
 143
 144   const exegesis::LLVMState *State;
 145   std::string LastError;
 146   llvm::raw_string_ostream ErrorStream;
 147 };
 148
 149 // Defining YAML traits for IO.
 150 namespace llvm {
 151 namespace yaml {
 152
 153 static YamlContext &getTypedContext(void *Ctx) {
 154   return *reinterpret_cast<YamlContext *>(Ctx);
 155 }
 156
 157 // std::vector<llvm::MCInst> will be rendered as a list.
 158 template <> struct SequenceElementTraits<llvm::MCInst> {
 159   static const bool flow = false;
 160 };
 161
 162 template <> struct ScalarTraits<llvm::MCInst> {
 163
 164   static void output(const llvm::MCInst &Value, void *Ctx,
 165                      llvm::raw_ostream &Out) {
 166     getTypedContext(Ctx).serializeMCInst(Value, Out);
 167   }
 168
 169   static StringRef input(StringRef Scalar, void *Ctx, llvm::MCInst &Value) {
 170     YamlContext &Context = getTypedContext(Ctx);
 171     Context.deserializeMCInst(Scalar, Value);
 172     return Context.getLastError();
 173   }
 174
 175   // By default strings are quoted only when necessary.
 176   // We force the use of single quotes for uniformity.
 177   static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
 178
 179   static const bool flow = true;
 180 };
 181
 182 // std::vector<exegesis::Measure> will be rendered as a list.
 183 template <> struct SequenceElementTraits<exegesis::BenchmarkMeasure> {
 184   static const bool flow = false;
 185 };
 186
 187 // exegesis::Measure is rendererd as a flow instead of a list.
 188 // e.g. { "key": "the key", "value": 0123 }
 189 template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
 190   static void mapping(IO &Io, exegesis::BenchmarkMeasure &Obj) {
 191     Io.mapRequired("key", Obj.Key);
 192     if (!Io.outputting()) {
 193       // For backward compatibility, interpret debug_string as a key.
 194       Io.mapOptional("debug_string", Obj.Key);
 195     }
 196     Io.mapRequired("value", Obj.PerInstructionValue);
 197     Io.mapOptional("per_snippet_value", Obj.PerSnippetValue);
 198   }
 199   static const bool flow = true;
 200 };
 201
 202 template <>
 203 struct ScalarEnumerationTraits<exegesis::InstructionBenchmark::ModeE> {
 204   static void enumeration(IO &Io,
 205                           exegesis::InstructionBenchmark::ModeE &Value) {
 206     Io.enumCase(Value, "", exegesis::InstructionBenchmark::Unknown);
 207     Io.enumCase(Value, "latency", exegesis::InstructionBenchmark::Latency);
 208     Io.enumCase(Value, "uops", exegesis::InstructionBenchmark::Uops);
 209   }
 210 };
 211
 212 // std::vector<exegesis::RegisterValue> will be rendered as a list.
 213 template <> struct SequenceElementTraits<exegesis::RegisterValue> {
 214   static const bool flow = false;
 215 };
 216
 217 template <> struct ScalarTraits<exegesis::RegisterValue> {
 218   static constexpr const unsigned kRadix = 16;
 219   static constexpr const bool kSigned = false;
 220
 221   static void output(const exegesis::RegisterValue &RV, void *Ctx,
 222                      llvm::raw_ostream &Out) {
 223     YamlContext &Context = getTypedContext(Ctx);
 224     Out << Context.getRegName(RV.Register) << "=0x"
 225         << RV.Value.toString(kRadix, kSigned);
 226   }
 227
 228   static StringRef input(StringRef String, void *Ctx,
 229                          exegesis::RegisterValue &RV) {
 230     llvm::SmallVector<llvm::StringRef, 2> Pieces;
 231     String.split(Pieces, "=0x", /* MaxSplit */ -1,
 232                  /* KeepEmpty */ false);
 233     YamlContext &Context = getTypedContext(Ctx);
 234     if (Pieces.size() == 2) {
 235       RV.Register = Context.getRegNo(Pieces[0]);
 236       const unsigned BitsNeeded = llvm::APInt::getBitsNeeded(Pieces[1], kRadix);
 237       RV.Value = llvm::APInt(BitsNeeded, Pieces[1], kRadix);
 238     } else {
 239       Context.getErrorStream()
 240           << "Unknown initial register value: '" << String << "'";
 241     }
 242     return Context.getLastError();
 243   }
 244
 245   static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
 246
 247   static const bool flow = true;
 248 };
 249
 250 template <>
 251 struct MappingContextTraits<exegesis::InstructionBenchmarkKey, YamlContext> {
 252   static void mapping(IO &Io, exegesis::InstructionBenchmarkKey &Obj,
 253                       YamlContext &Context) {
 254     Io.setContext(&Context);
 255     Io.mapRequired("instructions", Obj.Instructions);
 256     Io.mapOptional("config", Obj.Config);
 257     Io.mapRequired("register_initial_values", Obj.RegisterInitialValues);
 258   }
 259 };
 260
 261 template <>
 262 struct MappingContextTraits<exegesis::InstructionBenchmark, YamlContext> {
 263   struct NormalizedBinary {
 264     NormalizedBinary(IO &io) {}
 265     NormalizedBinary(IO &, std::vector<uint8_t> &Data) : Binary(Data) {}
 266     std::vector<uint8_t> denormalize(IO &) {
 267       std::vector<uint8_t> Data;
 268       std::string Str;
 269       raw_string_ostream OSS(Str);
 270       Binary.writeAsBinary(OSS);
 271       OSS.flush();
 272       Data.assign(Str.begin(), Str.end());
 273       return Data;
 274     }
 275
 276     BinaryRef Binary;
 277   };
 278
 279   static void mapping(IO &Io, exegesis::InstructionBenchmark &Obj,
 280                       YamlContext &Context) {
 281     Io.mapRequired("mode", Obj.Mode);
 282     Io.mapRequired("key", Obj.Key, Context);
 283     Io.mapRequired("cpu_name", Obj.CpuName);
 284     Io.mapRequired("llvm_triple", Obj.LLVMTriple);
 285     Io.mapRequired("num_repetitions", Obj.NumRepetitions);
 286     Io.mapRequired("measurements", Obj.Measurements);
 287     Io.mapRequired("error", Obj.Error);
 288     Io.mapOptional("info", Obj.Info);
 289     // AssembledSnippet
 290     MappingNormalization<NormalizedBinary, std::vector<uint8_t>> BinaryString(
 291         Io, Obj.AssembledSnippet);
 292     Io.mapOptional("assembled_snippet", BinaryString->Binary);
 293   }
 294 };
 295
 296 } // namespace yaml
 297 } // namespace llvm
 298
 299 namespace exegesis {
 300
 301 llvm::Expected<InstructionBenchmark>
 302 InstructionBenchmark::readYaml(const LLVMState &State,
 303                                llvm::StringRef Filename) {
 304   if (auto ExpectedMemoryBuffer =
 305           llvm::errorOrToExpected(llvm::MemoryBuffer::getFile(Filename))) {
 306     llvm::yaml::Input Yin(*ExpectedMemoryBuffer.get());
 307     YamlContext Context(State);
 308     InstructionBenchmark Benchmark;
 309     if (Yin.setCurrentDocument())
 310       llvm::yaml::yamlize(Yin, Benchmark, /*unused*/ true, Context);
 311     if (!Context.getLastError().empty())
 312       return llvm::make_error<BenchmarkFailure>(Context.getLastError());
 313     return Benchmark;
 314   } else {
 315     return ExpectedMemoryBuffer.takeError();
 316   }
 317 }
 318
 319 llvm::Expected<std::vector<InstructionBenchmark>>
 320 InstructionBenchmark::readYamls(const LLVMState &State,
 321                                 llvm::StringRef Filename) {
 322   if (auto ExpectedMemoryBuffer =
 323           llvm::errorOrToExpected(llvm::MemoryBuffer::getFile(Filename))) {
 324     llvm::yaml::Input Yin(*ExpectedMemoryBuffer.get());
 325     YamlContext Context(State);
 326     std::vector<InstructionBenchmark> Benchmarks;
 327     while (Yin.setCurrentDocument()) {
 328       Benchmarks.emplace_back();
 329       yamlize(Yin, Benchmarks.back(), /*unused*/ true, Context);
 330       if (Yin.error())
 331         return llvm::errorCodeToError(Yin.error());
 332       if (!Context.getLastError().empty())
 333         return llvm::make_error<BenchmarkFailure>(Context.getLastError());
 334       Yin.nextDocument();
 335     }
 336     return Benchmarks;
 337   } else {
 338     return ExpectedMemoryBuffer.takeError();
 339   }
 340 }
 341
 342 void InstructionBenchmark::writeYamlTo(const LLVMState &State,
 343                                        llvm::raw_ostream &OS) {
 344   llvm::yaml::Output Yout(OS);
 345   YamlContext Context(State);
 346   Yout.beginDocuments();
 347   llvm::yaml::yamlize(Yout, *this, /*unused*/ true, Context);
 348   Yout.endDocuments();
 349 }
 350
 351 void InstructionBenchmark::readYamlFrom(const LLVMState &State,
 352                                         llvm::StringRef InputContent) {
 353   llvm::yaml::Input Yin(InputContent);
 354   YamlContext Context(State);
 355   if (Yin.setCurrentDocument())
 356     llvm::yaml::yamlize(Yin, *this, /*unused*/ true, Context);
 357 }
 358
 359 llvm::Error InstructionBenchmark::writeYaml(const LLVMState &State,
 360                                             const llvm::StringRef Filename) {
 361   if (Filename == "-") {
 362     writeYamlTo(State, llvm::outs());
 363   } else {
 364     int ResultFD = 0;
 365     if (auto E = llvm::errorCodeToError(
 366             openFileForWrite(Filename, ResultFD, llvm::sys::fs::CD_CreateAlways,
 367                              llvm::sys::fs::F_Text))) {
 368       return E;
 369     }
 370     llvm::raw_fd_ostream Ostr(ResultFD, true /*shouldClose*/);
 371     writeYamlTo(State, Ostr);
 372   }
 373   return llvm::Error::success();
 374 }
 375
 376 void PerInstructionStats::push(const BenchmarkMeasure &BM) {
 377   if (Key.empty())
 378     Key = BM.Key;
 379   assert(Key == BM.Key);
 380   ++NumValues;
 381   SumValues += BM.PerInstructionValue;
 382   MaxValue = std::max(MaxValue, BM.PerInstructionValue);
 383   MinValue = std::min(MinValue, BM.PerInstructionValue);
 384 }
 385
 386 } // namespace exegesis