third-party/benchmark/test/output_test_helper.cc

   1 #include <cstdio>
   2 #include <cstring>
   3 #include <fstream>
   4 #include <iostream>
   5 #include <map>
   6 #include <memory>
   7 #include <random>
   8 #include <sstream>
   9 #include <streambuf>
  10
  11 #include "../src/benchmark_api_internal.h"
  12 #include "../src/check.h"  // NOTE: check.h is for internal use only!
  13 #include "../src/log.h"    // NOTE: log.h is for internal use only
  14 #include "../src/re.h"     // NOTE: re.h is for internal use only
  15 #include "output_test.h"
  16
  17 // ========================================================================= //
  18 // ------------------------------ Internals -------------------------------- //
  19 // ========================================================================= //
  20 namespace internal {
  21 namespace {
  22
  23 using TestCaseList = std::vector<TestCase>;
  24
  25 // Use a vector because the order elements are added matters during iteration.
  26 // std::map/unordered_map don't guarantee that.
  27 // For example:
  28 //  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
  29 //     Substitute("%HelloWorld") // Always expands to Hello.
  30 using SubMap = std::vector<std::pair<std::string, std::string>>;
  31
  32 TestCaseList& GetTestCaseList(TestCaseID ID) {
  33   // Uses function-local statics to ensure initialization occurs
  34   // before first use.
  35   static TestCaseList lists[TC_NumID];
  36   return lists[ID];
  37 }
  38
  39 SubMap& GetSubstitutions() {
  40   // Don't use 'dec_re' from header because it may not yet be initialized.
  41   // clang-format off
  42   static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
  43   static std::string time_re = "([0-9]+[.])?[0-9]+";
  44   static std::string percentage_re = "[0-9]+[.][0-9]{2}";
  45   static SubMap map = {
  46       {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
  47       // human-readable float
  48       {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
  49       {"%percentage", percentage_re},
  50       {"%int", "[ ]*[0-9]+"},
  51       {" %s ", "[ ]+"},
  52       {"%time", "[ ]*" + time_re + "[ ]+ns"},
  53       {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
  54       {"%console_percentage_report", "[ ]*" + percentage_re + "[ ]+% [ ]*" + percentage_re + "[ ]+% [ ]*[0-9]+"},
  55       {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
  56       {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"},
  57       {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"},
  58       {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
  59       {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
  60       {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
  61       {"%csv_header",
  62        "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
  63        "items_per_second,label,error_occurred,error_message"},
  64       {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
  65       {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
  66       {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"},
  67       {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"},
  68       {"%csv_bytes_report",
  69        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
  70       {"%csv_items_report",
  71        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
  72       {"%csv_bytes_items_report",
  73        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
  74        "," + safe_dec_re + ",,,"},
  75       {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
  76       {"%csv_label_report_end", ",,"}};
  77   // clang-format on
  78   return map;
  79 }
  80
  81 std::string PerformSubstitutions(std::string source) {
  82   SubMap const& subs = GetSubstitutions();
  83   using SizeT = std::string::size_type;
  84   for (auto const& KV : subs) {
  85     SizeT pos;
  86     SizeT next_start = 0;
  87     while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
  88       next_start = pos + KV.second.size();
  89       source.replace(pos, KV.first.size(), KV.second);
  90     }
  91   }
  92   return source;
  93 }
  94
  95 void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
  96                TestCaseList const& not_checks) {
  97   std::string first_line;
  98   bool on_first = true;
  99   std::string line;
 100   while (remaining_output.eof() == false) {
 101     BM_CHECK(remaining_output.good());
 102     std::getline(remaining_output, line);
 103     if (on_first) {
 104       first_line = line;
 105       on_first = false;
 106     }
 107     for (const auto& NC : not_checks) {
 108       BM_CHECK(!NC.regex->Match(line))
 109           << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
 110           << NC.regex_str << "\""
 111           << "\n    actual regex string \"" << TC.substituted_regex << "\""
 112           << "\n    started matching near: " << first_line;
 113     }
 114     if (TC.regex->Match(line)) return;
 115     BM_CHECK(TC.match_rule != MR_Next)
 116         << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
 117         << "\""
 118         << "\n    actual regex string \"" << TC.substituted_regex << "\""
 119         << "\n    started matching near: " << first_line;
 120   }
 121   BM_CHECK(remaining_output.eof() == false)
 122       << "End of output reached before match for regex \"" << TC.regex_str
 123       << "\" was found"
 124       << "\n    actual regex string \"" << TC.substituted_regex << "\""
 125       << "\n    started matching near: " << first_line;
 126 }
 127
 128 void CheckCases(TestCaseList const& checks, std::stringstream& output) {
 129   std::vector<TestCase> not_checks;
 130   for (size_t i = 0; i < checks.size(); ++i) {
 131     const auto& TC = checks[i];
 132     if (TC.match_rule == MR_Not) {
 133       not_checks.push_back(TC);
 134       continue;
 135     }
 136     CheckCase(output, TC, not_checks);
 137     not_checks.clear();
 138   }
 139 }
 140
 141 class TestReporter : public benchmark::BenchmarkReporter {
 142  public:
 143   TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
 144       : reporters_(std::move(reps)) {}
 145
 146   virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
 147     bool last_ret = false;
 148     bool first = true;
 149     for (auto rep : reporters_) {
 150       bool new_ret = rep->ReportContext(context);
 151       BM_CHECK(first || new_ret == last_ret)
 152           << "Reports return different values for ReportContext";
 153       first = false;
 154       last_ret = new_ret;
 155     }
 156     (void)first;
 157     return last_ret;
 158   }
 159
 160   void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
 161     for (auto rep : reporters_) rep->ReportRuns(report);
 162   }
 163   void Finalize() BENCHMARK_OVERRIDE {
 164     for (auto rep : reporters_) rep->Finalize();
 165   }
 166
 167  private:
 168   std::vector<benchmark::BenchmarkReporter*> reporters_;
 169 };
 170 }  // namespace
 171
 172 }  // end namespace internal
 173
 174 // ========================================================================= //
 175 // -------------------------- Results checking ----------------------------- //
 176 // ========================================================================= //
 177
 178 namespace internal {
 179
 180 // Utility class to manage subscribers for checking benchmark results.
 181 // It works by parsing the CSV output to read the results.
 182 class ResultsChecker {
 183  public:
 184   struct PatternAndFn : public TestCase {  // reusing TestCase for its regexes
 185     PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
 186         : TestCase(rx), fn(std::move(fn_)) {}
 187     ResultsCheckFn fn;
 188   };
 189
 190   std::vector<PatternAndFn> check_patterns;
 191   std::vector<Results> results;
 192   std::vector<std::string> field_names;
 193
 194   void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);
 195
 196   void CheckResults(std::stringstream& output);
 197
 198  private:
 199   void SetHeader_(const std::string& csv_header);
 200   void SetValues_(const std::string& entry_csv_line);
 201
 202   std::vector<std::string> SplitCsv_(const std::string& line);
 203 };
 204
 205 // store the static ResultsChecker in a function to prevent initialization
 206 // order problems
 207 ResultsChecker& GetResultsChecker() {
 208   static ResultsChecker rc;
 209   return rc;
 210 }
 211
 212 // add a results checker for a benchmark
 213 void ResultsChecker::Add(const std::string& entry_pattern,
 214                          const ResultsCheckFn& fn) {
 215   check_patterns.emplace_back(entry_pattern, fn);
 216 }
 217
 218 // check the results of all subscribed benchmarks
 219 void ResultsChecker::CheckResults(std::stringstream& output) {
 220   // first reset the stream to the start
 221   {
 222     auto start = std::stringstream::pos_type(0);
 223     // clear before calling tellg()
 224     output.clear();
 225     // seek to zero only when needed
 226     if (output.tellg() > start) output.seekg(start);
 227     // and just in case
 228     output.clear();
 229   }
 230   // now go over every line and publish it to the ResultsChecker
 231   std::string line;
 232   bool on_first = true;
 233   while (output.eof() == false) {
 234     BM_CHECK(output.good());
 235     std::getline(output, line);
 236     if (on_first) {
 237       SetHeader_(line);  // this is important
 238       on_first = false;
 239       continue;
 240     }
 241     SetValues_(line);
 242   }
 243   // finally we can call the subscribed check functions
 244   for (const auto& p : check_patterns) {
 245     BM_VLOG(2) << "--------------------------------\n";
 246     BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
 247     for (const auto& r : results) {
 248       if (!p.regex->Match(r.name)) {
 249         BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
 250         continue;
 251       } else {
 252         BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
 253       }
 254       BM_VLOG(1) << "Checking results of " << r.name << ": ... \n";
 255       p.fn(r);
 256       BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n";
 257     }
 258   }
 259 }
 260
 261 // prepare for the names in this header
 262 void ResultsChecker::SetHeader_(const std::string& csv_header) {
 263   field_names = SplitCsv_(csv_header);
 264 }
 265
 266 // set the values for a benchmark
 267 void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
 268   if (entry_csv_line.empty()) return;  // some lines are empty
 269   BM_CHECK(!field_names.empty());
 270   auto vals = SplitCsv_(entry_csv_line);
 271   BM_CHECK_EQ(vals.size(), field_names.size());
 272   results.emplace_back(vals[0]);  // vals[0] is the benchmark name
 273   auto& entry = results.back();
 274   for (size_t i = 1, e = vals.size(); i < e; ++i) {
 275     entry.values[field_names[i]] = vals[i];
 276   }
 277 }
 278
 279 // a quick'n'dirty csv splitter (eliminating quotes)
 280 std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
 281   std::vector<std::string> out;
 282   if (line.empty()) return out;
 283   if (!field_names.empty()) out.reserve(field_names.size());
 284   size_t prev = 0, pos = line.find_first_of(','), curr = pos;
 285   while (pos != line.npos) {
 286     BM_CHECK(curr > 0);
 287     if (line[prev] == '"') ++prev;
 288     if (line[curr - 1] == '"') --curr;
 289     out.push_back(line.substr(prev, curr - prev));
 290     prev = pos + 1;
 291     pos = line.find_first_of(',', pos + 1);
 292     curr = pos;
 293   }
 294   curr = line.size();
 295   if (line[prev] == '"') ++prev;
 296   if (line[curr - 1] == '"') --curr;
 297   out.push_back(line.substr(prev, curr - prev));
 298   return out;
 299 }
 300
 301 }  // end namespace internal
 302
 303 size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) {
 304   auto& rc = internal::GetResultsChecker();
 305   rc.Add(bm_name, fn);
 306   return rc.results.size();
 307 }
 308
 309 int Results::NumThreads() const {
 310   auto pos = name.find("/threads:");
 311   if (pos == name.npos) return 1;
 312   auto end = name.find('/', pos + 9);
 313   std::stringstream ss;
 314   ss << name.substr(pos + 9, end);
 315   int num = 1;
 316   ss >> num;
 317   BM_CHECK(!ss.fail());
 318   return num;
 319 }
 320
 321 double Results::NumIterations() const { return GetAs<double>("iterations"); }
 322
 323 double Results::GetTime(BenchmarkTime which) const {
 324   BM_CHECK(which == kCpuTime || which == kRealTime);
 325   const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
 326   double val = GetAs<double>(which_str);
 327   auto unit = Get("time_unit");
 328   BM_CHECK(unit);
 329   if (*unit == "ns") {
 330     return val * 1.e-9;
 331   } else if (*unit == "us") {
 332     return val * 1.e-6;
 333   } else if (*unit == "ms") {
 334     return val * 1.e-3;
 335   } else if (*unit == "s") {
 336     return val;
 337   } else {
 338     BM_CHECK(1 == 0) << "unknown time unit: " << *unit;
 339     return 0;
 340   }
 341 }
 342
 343 // ========================================================================= //
 344 // -------------------------- Public API Definitions------------------------ //
 345 // ========================================================================= //
 346
 347 TestCase::TestCase(std::string re, int rule)
 348     : regex_str(std::move(re)),
 349       match_rule(rule),
 350       substituted_regex(internal::PerformSubstitutions(regex_str)),
 351       regex(std::make_shared<benchmark::Regex>()) {
 352   std::string err_str;
 353   regex->Init(substituted_regex, &err_str);
 354   BM_CHECK(err_str.empty())
 355       << "Could not construct regex \"" << substituted_regex << "\""
 356       << "\n    originally \"" << regex_str << "\""
 357       << "\n    got error: " << err_str;
 358 }
 359
 360 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
 361   auto& L = internal::GetTestCaseList(ID);
 362   L.insert(L.end(), il);
 363   return 0;
 364 }
 365
 366 int SetSubstitutions(
 367     std::initializer_list<std::pair<std::string, std::string>> il) {
 368   auto& subs = internal::GetSubstitutions();
 369   for (auto KV : il) {
 370     bool exists = false;
 371     KV.second = internal::PerformSubstitutions(KV.second);
 372     for (auto& EKV : subs) {
 373       if (EKV.first == KV.first) {
 374         EKV.second = std::move(KV.second);
 375         exists = true;
 376         break;
 377       }
 378     }
 379     if (!exists) subs.push_back(std::move(KV));
 380   }
 381   return 0;
 382 }
 383
 384 // Disable deprecated warnings temporarily because we need to reference
 385 // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
 386 BENCHMARK_DISABLE_DEPRECATED_WARNING
 387
 388 void RunOutputTests(int argc, char* argv[]) {
 389   using internal::GetTestCaseList;
 390   benchmark::Initialize(&argc, argv);
 391   auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
 392   benchmark::ConsoleReporter CR(options);
 393   benchmark::JSONReporter JR;
 394   benchmark::CSVReporter CSVR;
 395   struct ReporterTest {
 396     const char* name;
 397     std::vector<TestCase>& output_cases;
 398     std::vector<TestCase>& error_cases;
 399     benchmark::BenchmarkReporter& reporter;
 400     std::stringstream out_stream;
 401     std::stringstream err_stream;
 402
 403     ReporterTest(const char* n, std::vector<TestCase>& out_tc,
 404                  std::vector<TestCase>& err_tc,
 405                  benchmark::BenchmarkReporter& br)
 406         : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
 407       reporter.SetOutputStream(&out_stream);
 408       reporter.SetErrorStream(&err_stream);
 409     }
 410   } TestCases[] = {
 411       {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
 412        GetTestCaseList(TC_ConsoleErr), CR},
 413       {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
 414        JR},
 415       {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
 416        CSVR},
 417   };
 418
 419   // Create the test reporter and run the benchmarks.
 420   std::cout << "Running benchmarks...\n";
 421   internal::TestReporter test_rep({&CR, &JR, &CSVR});
 422   benchmark::RunSpecifiedBenchmarks(&test_rep);
 423
 424   for (auto& rep_test : TestCases) {
 425     std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
 426     std::string banner(msg.size() - 1, '-');
 427     std::cout << banner << msg << banner << "\n";
 428
 429     std::cerr << rep_test.err_stream.str();
 430     std::cout << rep_test.out_stream.str();
 431
 432     internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
 433     internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
 434
 435     std::cout << "\n";
 436   }
 437
 438   // now that we know the output is as expected, we can dispatch
 439   // the checks to subscribees.
 440   auto& csv = TestCases[2];
 441   // would use == but gcc spits a warning
 442   BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
 443   internal::GetResultsChecker().CheckResults(csv.out_stream);
 444 }
 445
 446 BENCHMARK_RESTORE_DEPRECATED_WARNING
 447
 448 int SubstrCnt(const std::string& haystack, const std::string& pat) {
 449   if (pat.length() == 0) return 0;
 450   int count = 0;
 451   for (size_t offset = haystack.find(pat); offset != std::string::npos;
 452        offset = haystack.find(pat, offset + pat.length()))
 453     ++count;
 454   return count;
 455 }
 456
 457 static char ToHex(int ch) {
 458   return ch < 10 ? static_cast<char>('0' + ch)
 459                  : static_cast<char>('a' + (ch - 10));
 460 }
 461
 462 static char RandomHexChar() {
 463   static std::mt19937 rd{std::random_device{}()};
 464   static std::uniform_int_distribution<int> mrand{0, 15};
 465   return ToHex(mrand(rd));
 466 }
 467
 468 static std::string GetRandomFileName() {
 469   std::string model = "test.%%%%%%";
 470   for (auto& ch : model) {
 471     if (ch == '%') ch = RandomHexChar();
 472   }
 473   return model;
 474 }
 475
 476 static bool FileExists(std::string const& name) {
 477   std::ifstream in(name.c_str());
 478   return in.good();
 479 }
 480
 481 static std::string GetTempFileName() {
 482   // This function attempts to avoid race conditions where two tests
 483   // create the same file at the same time. However, it still introduces races
 484   // similar to tmpnam.
 485   int retries = 3;
 486   while (--retries) {
 487     std::string name = GetRandomFileName();
 488     if (!FileExists(name)) return name;
 489   }
 490   std::cerr << "Failed to create unique temporary file name" << std::endl;
 491   std::abort();
 492 }
 493
 494 std::string GetFileReporterOutput(int argc, char* argv[]) {
 495   std::vector<char*> new_argv(argv, argv + argc);
 496   assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
 497
 498   std::string tmp_file_name = GetTempFileName();
 499   std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
 500
 501   std::string tmp = "--benchmark_out=";
 502   tmp += tmp_file_name;
 503   new_argv.emplace_back(const_cast<char*>(tmp.c_str()));
 504
 505   argc = int(new_argv.size());
 506
 507   benchmark::Initialize(&argc, new_argv.data());
 508   benchmark::RunSpecifiedBenchmarks();
 509
 510   // Read the output back from the file, and delete the file.
 511   std::ifstream tmp_stream(tmp_file_name);
 512   std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
 513                                    std::istreambuf_iterator<char>());
 514   std::remove(tmp_file_name.c_str());
 515
 516   return output;
 517 }