third-party/benchmark/test/output_test_helper.cc

   1 #include <cstdio>
   2 #include <cstring>
   3 #include <fstream>
   4 #include <iostream>
   5 #include <map>
   6 #include <memory>
   7 #include <random>
   8 #include <sstream>
   9 #include <streambuf>
  10
  11 #include "../src/benchmark_api_internal.h"
  12 #include "../src/check.h"  // NOTE: check.h is for internal use only!
  13 #include "../src/log.h"    // NOTE: log.h is for internal use only
  14 #include "../src/re.h"     // NOTE: re.h is for internal use only
  15 #include "output_test.h"
  16
  17 // ========================================================================= //
  18 // ------------------------------ Internals -------------------------------- //
  19 // ========================================================================= //
  20 namespace internal {
  21 namespace {
  22
  23 using TestCaseList = std::vector<TestCase>;
  24
  25 // Use a vector because the order elements are added matters during iteration.
  26 // std::map/unordered_map don't guarantee that.
  27 // For example:
  28 //  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
  29 //     Substitute("%HelloWorld") // Always expands to Hello.
  30 using SubMap = std::vector<std::pair<std::string, std::string>>;
  31
  32 TestCaseList& GetTestCaseList(TestCaseID ID) {
  33   // Uses function-local statics to ensure initialization occurs
  34   // before first use.
  35   static TestCaseList lists[TC_NumID];
  36   return lists[ID];
  37 }
  38
  39 SubMap& GetSubstitutions() {
  40   // Don't use 'dec_re' from header because it may not yet be initialized.
  41   // clang-format off
  42   static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
  43   static std::string time_re = "([0-9]+[.])?[0-9]+";
  44   static std::string percentage_re = "[0-9]+[.][0-9]{2}";
  45   static SubMap map = {
  46       {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
  47       // human-readable float
  48       {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"},
  49       {"%percentage", percentage_re},
  50       {"%int", "[ ]*[0-9]+"},
  51       {" %s ", "[ ]+"},
  52       {"%time", "[ ]*" + time_re + "[ ]+ns"},
  53       {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
  54       {"%console_percentage_report", "[ ]*" + percentage_re + "[ ]+% [ ]*" + percentage_re + "[ ]+% [ ]*[0-9]+"},
  55       {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
  56       {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"},
  57       {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"},
  58       {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
  59       {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
  60       {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
  61       {"%csv_header",
  62        "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
  63        "items_per_second,label,error_occurred,error_message"},
  64       {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
  65       {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
  66       {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"},
  67       {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"},
  68       {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"},
  69       {"%csv_bytes_report",
  70        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
  71       {"%csv_items_report",
  72        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
  73       {"%csv_bytes_items_report",
  74        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
  75        "," + safe_dec_re + ",,,"},
  76       {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
  77       {"%csv_label_report_end", ",,"}};
  78   // clang-format on
  79   return map;
  80 }
  81
  82 std::string PerformSubstitutions(std::string source) {
  83   SubMap const& subs = GetSubstitutions();
  84   using SizeT = std::string::size_type;
  85   for (auto const& KV : subs) {
  86     SizeT pos;
  87     SizeT next_start = 0;
  88     while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
  89       next_start = pos + KV.second.size();
  90       source.replace(pos, KV.first.size(), KV.second);
  91     }
  92   }
  93   return source;
  94 }
  95
  96 void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
  97                TestCaseList const& not_checks) {
  98   std::string first_line;
  99   bool on_first = true;
 100   std::string line;
 101   while (remaining_output.eof() == false) {
 102     BM_CHECK(remaining_output.good());
 103     std::getline(remaining_output, line);
 104     if (on_first) {
 105       first_line = line;
 106       on_first = false;
 107     }
 108     for (const auto& NC : not_checks) {
 109       BM_CHECK(!NC.regex->Match(line))
 110           << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
 111           << NC.regex_str << "\""
 112           << "\n    actual regex string \"" << TC.substituted_regex << "\""
 113           << "\n    started matching near: " << first_line;
 114     }
 115     if (TC.regex->Match(line)) return;
 116     BM_CHECK(TC.match_rule != MR_Next)
 117         << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
 118         << "\""
 119         << "\n    actual regex string \"" << TC.substituted_regex << "\""
 120         << "\n    started matching near: " << first_line;
 121   }
 122   BM_CHECK(remaining_output.eof() == false)
 123       << "End of output reached before match for regex \"" << TC.regex_str
 124       << "\" was found"
 125       << "\n    actual regex string \"" << TC.substituted_regex << "\""
 126       << "\n    started matching near: " << first_line;
 127 }
 128
 129 void CheckCases(TestCaseList const& checks, std::stringstream& output) {
 130   std::vector<TestCase> not_checks;
 131   for (size_t i = 0; i < checks.size(); ++i) {
 132     const auto& TC = checks[i];
 133     if (TC.match_rule == MR_Not) {
 134       not_checks.push_back(TC);
 135       continue;
 136     }
 137     CheckCase(output, TC, not_checks);
 138     not_checks.clear();
 139   }
 140 }
 141
 142 class TestReporter : public benchmark::BenchmarkReporter {
 143  public:
 144   TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
 145       : reporters_(std::move(reps)) {}
 146
 147   bool ReportContext(const Context& context) override {
 148     bool last_ret = false;
 149     bool first = true;
 150     for (auto rep : reporters_) {
 151       bool new_ret = rep->ReportContext(context);
 152       BM_CHECK(first || new_ret == last_ret)
 153           << "Reports return different values for ReportContext";
 154       first = false;
 155       last_ret = new_ret;
 156     }
 157     (void)first;
 158     return last_ret;
 159   }
 160
 161   void ReportRuns(const std::vector<Run>& report) override {
 162     for (auto rep : reporters_) rep->ReportRuns(report);
 163   }
 164   void Finalize() override {
 165     for (auto rep : reporters_) rep->Finalize();
 166   }
 167
 168  private:
 169   std::vector<benchmark::BenchmarkReporter*> reporters_;
 170 };
 171 }  // namespace
 172
 173 }  // end namespace internal
 174
 175 // ========================================================================= //
 176 // -------------------------- Results checking ----------------------------- //
 177 // ========================================================================= //
 178
 179 namespace internal {
 180
 181 // Utility class to manage subscribers for checking benchmark results.
 182 // It works by parsing the CSV output to read the results.
 183 class ResultsChecker {
 184  public:
 185   struct PatternAndFn : public TestCase {  // reusing TestCase for its regexes
 186     PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
 187         : TestCase(rx), fn(std::move(fn_)) {}
 188     ResultsCheckFn fn;
 189   };
 190
 191   std::vector<PatternAndFn> check_patterns;
 192   std::vector<Results> results;
 193   std::vector<std::string> field_names;
 194
 195   void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);
 196
 197   void CheckResults(std::stringstream& output);
 198
 199  private:
 200   void SetHeader_(const std::string& csv_header);
 201   void SetValues_(const std::string& entry_csv_line);
 202
 203   std::vector<std::string> SplitCsv_(const std::string& line);
 204 };
 205
 206 // store the static ResultsChecker in a function to prevent initialization
 207 // order problems
 208 ResultsChecker& GetResultsChecker() {
 209   static ResultsChecker rc;
 210   return rc;
 211 }
 212
 213 // add a results checker for a benchmark
 214 void ResultsChecker::Add(const std::string& entry_pattern,
 215                          const ResultsCheckFn& fn) {
 216   check_patterns.emplace_back(entry_pattern, fn);
 217 }
 218
 219 // check the results of all subscribed benchmarks
 220 void ResultsChecker::CheckResults(std::stringstream& output) {
 221   // first reset the stream to the start
 222   {
 223     auto start = std::stringstream::pos_type(0);
 224     // clear before calling tellg()
 225     output.clear();
 226     // seek to zero only when needed
 227     if (output.tellg() > start) output.seekg(start);
 228     // and just in case
 229     output.clear();
 230   }
 231   // now go over every line and publish it to the ResultsChecker
 232   std::string line;
 233   bool on_first = true;
 234   while (output.eof() == false) {
 235     BM_CHECK(output.good());
 236     std::getline(output, line);
 237     if (on_first) {
 238       SetHeader_(line);  // this is important
 239       on_first = false;
 240       continue;
 241     }
 242     SetValues_(line);
 243   }
 244   // finally we can call the subscribed check functions
 245   for (const auto& p : check_patterns) {
 246     BM_VLOG(2) << "--------------------------------\n";
 247     BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
 248     for (const auto& r : results) {
 249       if (!p.regex->Match(r.name)) {
 250         BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
 251         continue;
 252       }
 253       BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
 254       BM_VLOG(1) << "Checking results of " << r.name << ": ... \n";
 255       p.fn(r);
 256       BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n";
 257     }
 258   }
 259 }
 260
 261 // prepare for the names in this header
 262 void ResultsChecker::SetHeader_(const std::string& csv_header) {
 263   field_names = SplitCsv_(csv_header);
 264 }
 265
 266 // set the values for a benchmark
 267 void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
 268   if (entry_csv_line.empty()) return;  // some lines are empty
 269   BM_CHECK(!field_names.empty());
 270   auto vals = SplitCsv_(entry_csv_line);
 271   BM_CHECK_EQ(vals.size(), field_names.size());
 272   results.emplace_back(vals[0]);  // vals[0] is the benchmark name
 273   auto& entry = results.back();
 274   for (size_t i = 1, e = vals.size(); i < e; ++i) {
 275     entry.values[field_names[i]] = vals[i];
 276   }
 277 }
 278
 279 // a quick'n'dirty csv splitter (eliminating quotes)
 280 std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
 281   std::vector<std::string> out;
 282   if (line.empty()) return out;
 283   if (!field_names.empty()) out.reserve(field_names.size());
 284   size_t prev = 0, pos = line.find_first_of(','), curr = pos;
 285   while (pos != line.npos) {
 286     BM_CHECK(curr > 0);
 287     if (line[prev] == '"') ++prev;
 288     if (line[curr - 1] == '"') --curr;
 289     out.push_back(line.substr(prev, curr - prev));
 290     prev = pos + 1;
 291     pos = line.find_first_of(',', pos + 1);
 292     curr = pos;
 293   }
 294   curr = line.size();
 295   if (line[prev] == '"') ++prev;
 296   if (line[curr - 1] == '"') --curr;
 297   out.push_back(line.substr(prev, curr - prev));
 298   return out;
 299 }
 300
 301 }  // end namespace internal
 302
 303 size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) {
 304   auto& rc = internal::GetResultsChecker();
 305   rc.Add(bm_name, fn);
 306   return rc.results.size();
 307 }
 308
 309 int Results::NumThreads() const {
 310   auto pos = name.find("/threads:");
 311   if (pos == name.npos) return 1;
 312   auto end = name.find('/', pos + 9);
 313   std::stringstream ss;
 314   ss << name.substr(pos + 9, end);
 315   int num = 1;
 316   ss >> num;
 317   BM_CHECK(!ss.fail());
 318   return num;
 319 }
 320
 321 double Results::NumIterations() const { return GetAs<double>("iterations"); }
 322
 323 double Results::GetTime(BenchmarkTime which) const {
 324   BM_CHECK(which == kCpuTime || which == kRealTime);
 325   const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
 326   double val = GetAs<double>(which_str);
 327   auto unit = Get("time_unit");
 328   BM_CHECK(unit);
 329   if (*unit == "ns") {
 330     return val * 1.e-9;
 331   }
 332   if (*unit == "us") {
 333     return val * 1.e-6;
 334   }
 335   if (*unit == "ms") {
 336     return val * 1.e-3;
 337   }
 338   if (*unit == "s") {
 339     return val;
 340   }
 341   BM_CHECK(1 == 0) << "unknown time unit: " << *unit;
 342   return 0;
 343 }
 344
 345 // ========================================================================= //
 346 // -------------------------- Public API Definitions------------------------ //
 347 // ========================================================================= //
 348
 349 TestCase::TestCase(std::string re, int rule)
 350     : regex_str(std::move(re)),
 351       match_rule(rule),
 352       substituted_regex(internal::PerformSubstitutions(regex_str)),
 353       regex(std::make_shared<benchmark::Regex>()) {
 354   std::string err_str;
 355   regex->Init(substituted_regex, &err_str);
 356   BM_CHECK(err_str.empty())
 357       << "Could not construct regex \"" << substituted_regex << "\""
 358       << "\n    originally \"" << regex_str << "\""
 359       << "\n    got error: " << err_str;
 360 }
 361
 362 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
 363   auto& L = internal::GetTestCaseList(ID);
 364   L.insert(L.end(), il);
 365   return 0;
 366 }
 367
 368 int SetSubstitutions(
 369     std::initializer_list<std::pair<std::string, std::string>> il) {
 370   auto& subs = internal::GetSubstitutions();
 371   for (auto KV : il) {
 372     bool exists = false;
 373     KV.second = internal::PerformSubstitutions(KV.second);
 374     for (auto& EKV : subs) {
 375       if (EKV.first == KV.first) {
 376         EKV.second = std::move(KV.second);
 377         exists = true;
 378         break;
 379       }
 380     }
 381     if (!exists) subs.push_back(std::move(KV));
 382   }
 383   return 0;
 384 }
 385
 386 // Disable deprecated warnings temporarily because we need to reference
 387 // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
 388 BENCHMARK_DISABLE_DEPRECATED_WARNING
 389
 390 void RunOutputTests(int argc, char* argv[]) {
 391   using internal::GetTestCaseList;
 392   benchmark::Initialize(&argc, argv);
 393   auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
 394   benchmark::ConsoleReporter CR(options);
 395   benchmark::JSONReporter JR;
 396   benchmark::CSVReporter CSVR;
 397   struct ReporterTest {
 398     std::string name;
 399     std::vector<TestCase>& output_cases;
 400     std::vector<TestCase>& error_cases;
 401     benchmark::BenchmarkReporter& reporter;
 402     std::stringstream out_stream;
 403     std::stringstream err_stream;
 404
 405     ReporterTest(const std::string& n, std::vector<TestCase>& out_tc,
 406                  std::vector<TestCase>& err_tc,
 407                  benchmark::BenchmarkReporter& br)
 408         : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
 409       reporter.SetOutputStream(&out_stream);
 410       reporter.SetErrorStream(&err_stream);
 411     }
 412   } TestCases[] = {
 413       {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut),
 414        GetTestCaseList(TC_ConsoleErr), CR},
 415       {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut),
 416        GetTestCaseList(TC_JSONErr), JR},
 417       {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut),
 418        GetTestCaseList(TC_CSVErr), CSVR},
 419   };
 420
 421   // Create the test reporter and run the benchmarks.
 422   std::cout << "Running benchmarks...\n";
 423   internal::TestReporter test_rep({&CR, &JR, &CSVR});
 424   benchmark::RunSpecifiedBenchmarks(&test_rep);
 425
 426   for (auto& rep_test : TestCases) {
 427     std::string msg =
 428         std::string("\nTesting ") + rep_test.name + std::string(" Output\n");
 429     std::string banner(msg.size() - 1, '-');
 430     std::cout << banner << msg << banner << "\n";
 431
 432     std::cerr << rep_test.err_stream.str();
 433     std::cout << rep_test.out_stream.str();
 434
 435     internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
 436     internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
 437
 438     std::cout << "\n";
 439   }
 440
 441   // now that we know the output is as expected, we can dispatch
 442   // the checks to subscribees.
 443   auto& csv = TestCases[2];
 444   // would use == but gcc spits a warning
 445   BM_CHECK(csv.name == std::string("CSVReporter"));
 446   internal::GetResultsChecker().CheckResults(csv.out_stream);
 447 }
 448
 449 BENCHMARK_RESTORE_DEPRECATED_WARNING
 450
 451 int SubstrCnt(const std::string& haystack, const std::string& pat) {
 452   if (pat.length() == 0) return 0;
 453   int count = 0;
 454   for (size_t offset = haystack.find(pat); offset != std::string::npos;
 455        offset = haystack.find(pat, offset + pat.length()))
 456     ++count;
 457   return count;
 458 }
 459
 460 static char ToHex(int ch) {
 461   return ch < 10 ? static_cast<char>('0' + ch)
 462                  : static_cast<char>('a' + (ch - 10));
 463 }
 464
 465 static char RandomHexChar() {
 466   static std::mt19937 rd{std::random_device{}()};
 467   static std::uniform_int_distribution<int> mrand{0, 15};
 468   return ToHex(mrand(rd));
 469 }
 470
 471 static std::string GetRandomFileName() {
 472   std::string model = "test.%%%%%%";
 473   for (auto& ch : model) {
 474     if (ch == '%') ch = RandomHexChar();
 475   }
 476   return model;
 477 }
 478
 479 static bool FileExists(std::string const& name) {
 480   std::ifstream in(name.c_str());
 481   return in.good();
 482 }
 483
 484 static std::string GetTempFileName() {
 485   // This function attempts to avoid race conditions where two tests
 486   // create the same file at the same time. However, it still introduces races
 487   // similar to tmpnam.
 488   int retries = 3;
 489   while (--retries) {
 490     std::string name = GetRandomFileName();
 491     if (!FileExists(name)) return name;
 492   }
 493   std::cerr << "Failed to create unique temporary file name" << std::endl;
 494   std::abort();
 495 }
 496
 497 std::string GetFileReporterOutput(int argc, char* argv[]) {
 498   std::vector<char*> new_argv(argv, argv + argc);
 499   assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
 500
 501   std::string tmp_file_name = GetTempFileName();
 502   std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
 503
 504   std::string tmp = "--benchmark_out=";
 505   tmp += tmp_file_name;
 506   new_argv.emplace_back(const_cast<char*>(tmp.c_str()));
 507
 508   argc = int(new_argv.size());
 509
 510   benchmark::Initialize(&argc, new_argv.data());
 511   benchmark::RunSpecifiedBenchmarks();
 512
 513   // Read the output back from the file, and delete the file.
 514   std::ifstream tmp_stream(tmp_file_name);
 515   std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
 516                                    std::istreambuf_iterator<char>());
 517   std::remove(tmp_file_name.c_str());
 518
 519   return output;
 520 }