utils/benchmark/test/output_test_helper.cc

   1 #include <iostream>
   2 #include <map>
   3 #include <memory>
   4 #include <sstream>
   5 #include <cstring>
   6
   7 #include "../src/check.h"  // NOTE: check.h is for internal use only!
   8 #include "../src/re.h"     // NOTE: re.h is for internal use only
   9 #include "output_test.h"
  10 #include "../src/benchmark_api_internal.h"
  11
  12 // ========================================================================= //
  13 // ------------------------------ Internals -------------------------------- //
  14 // ========================================================================= //
  15 namespace internal {
  16 namespace {
  17
  18 using TestCaseList = std::vector<TestCase>;
  19
  20 // Use a vector because the order elements are added matters during iteration.
  21 // std::map/unordered_map don't guarantee that.
  22 // For example:
  23 //  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
  24 //     Substitute("%HelloWorld") // Always expands to Hello.
  25 using SubMap = std::vector<std::pair<std::string, std::string>>;
  26
  27 TestCaseList& GetTestCaseList(TestCaseID ID) {
  28   // Uses function-local statics to ensure initialization occurs
  29   // before first use.
  30   static TestCaseList lists[TC_NumID];
  31   return lists[ID];
  32 }
  33
  34 SubMap& GetSubstitutions() {
  35   // Don't use 'dec_re' from header because it may not yet be initialized.
  36   static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
  37   static SubMap map = {
  38       {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
  39       // human-readable float
  40       {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
  41       {"%int", "[ ]*[0-9]+"},
  42       {" %s ", "[ ]+"},
  43       {"%time", "[ ]*[0-9]{1,6} ns"},
  44       {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"},
  45       {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
  46       {"%csv_header",
  47        "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
  48        "items_per_second,label,error_occurred,error_message"},
  49       {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
  50       {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
  51       {"%csv_bytes_report",
  52        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
  53       {"%csv_items_report",
  54        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
  55       {"%csv_bytes_items_report",
  56        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
  57        "," + safe_dec_re + ",,,"},
  58       {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
  59       {"%csv_label_report_end", ",,"}};
  60   return map;
  61 }
  62
  63 std::string PerformSubstitutions(std::string source) {
  64   SubMap const& subs = GetSubstitutions();
  65   using SizeT = std::string::size_type;
  66   for (auto const& KV : subs) {
  67     SizeT pos;
  68     SizeT next_start = 0;
  69     while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
  70       next_start = pos + KV.second.size();
  71       source.replace(pos, KV.first.size(), KV.second);
  72     }
  73   }
  74   return source;
  75 }
  76
  77 void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
  78                TestCaseList const& not_checks) {
  79   std::string first_line;
  80   bool on_first = true;
  81   std::string line;
  82   while (remaining_output.eof() == false) {
  83     CHECK(remaining_output.good());
  84     std::getline(remaining_output, line);
  85     if (on_first) {
  86       first_line = line;
  87       on_first = false;
  88     }
  89     for (const auto& NC : not_checks) {
  90       CHECK(!NC.regex->Match(line))
  91           << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
  92           << NC.regex_str << "\""
  93           << "\n    actual regex string \"" << TC.substituted_regex << "\""
  94           << "\n    started matching near: " << first_line;
  95     }
  96     if (TC.regex->Match(line)) return;
  97     CHECK(TC.match_rule != MR_Next)
  98         << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
  99         << "\""
 100         << "\n    actual regex string \"" << TC.substituted_regex << "\""
 101         << "\n    started matching near: " << first_line;
 102   }
 103   CHECK(remaining_output.eof() == false)
 104       << "End of output reached before match for regex \"" << TC.regex_str
 105       << "\" was found"
 106       << "\n    actual regex string \"" << TC.substituted_regex << "\""
 107       << "\n    started matching near: " << first_line;
 108 }
 109
 110 void CheckCases(TestCaseList const& checks, std::stringstream& output) {
 111   std::vector<TestCase> not_checks;
 112   for (size_t i = 0; i < checks.size(); ++i) {
 113     const auto& TC = checks[i];
 114     if (TC.match_rule == MR_Not) {
 115       not_checks.push_back(TC);
 116       continue;
 117     }
 118     CheckCase(output, TC, not_checks);
 119     not_checks.clear();
 120   }
 121 }
 122
 123 class TestReporter : public benchmark::BenchmarkReporter {
 124  public:
 125   TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
 126       : reporters_(reps) {}
 127
 128   virtual bool ReportContext(const Context& context) {
 129     bool last_ret = false;
 130     bool first = true;
 131     for (auto rep : reporters_) {
 132       bool new_ret = rep->ReportContext(context);
 133       CHECK(first || new_ret == last_ret)
 134           << "Reports return different values for ReportContext";
 135       first = false;
 136       last_ret = new_ret;
 137     }
 138     (void)first;
 139     return last_ret;
 140   }
 141
 142   void ReportRuns(const std::vector<Run>& report) {
 143     for (auto rep : reporters_) rep->ReportRuns(report);
 144   }
 145   void Finalize() {
 146     for (auto rep : reporters_) rep->Finalize();
 147   }
 148
 149  private:
 150   std::vector<benchmark::BenchmarkReporter *> reporters_;
 151 };
 152 }
 153
 154 }  // end namespace internal
 155
 156 // ========================================================================= //
 157 // -------------------------- Results checking ----------------------------- //
 158 // ========================================================================= //
 159
 160 namespace internal {
 161
 162 // Utility class to manage subscribers for checking benchmark results.
 163 // It works by parsing the CSV output to read the results.
 164 class ResultsChecker {
 165  public:
 166
 167   struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
 168     PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
 169     : TestCase(rx), fn(fn_) {}
 170     ResultsCheckFn fn;
 171   };
 172
 173   std::vector< PatternAndFn > check_patterns;
 174   std::vector< Results > results;
 175   std::vector< std::string > field_names;
 176
 177   void Add(const std::string& entry_pattern, ResultsCheckFn fn);
 178
 179   void CheckResults(std::stringstream& output);
 180
 181  private:
 182
 183   void SetHeader_(const std::string& csv_header);
 184   void SetValues_(const std::string& entry_csv_line);
 185
 186   std::vector< std::string > SplitCsv_(const std::string& line);
 187
 188 };
 189
 190 // store the static ResultsChecker in a function to prevent initialization
 191 // order problems
 192 ResultsChecker& GetResultsChecker() {
 193   static ResultsChecker rc;
 194   return rc;
 195 }
 196
 197 // add a results checker for a benchmark
 198 void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
 199   check_patterns.emplace_back(entry_pattern, fn);
 200 }
 201
 202 // check the results of all subscribed benchmarks
 203 void ResultsChecker::CheckResults(std::stringstream& output) {
 204   // first reset the stream to the start
 205   {
 206     auto start = std::ios::streampos(0);
 207     // clear before calling tellg()
 208     output.clear();
 209     // seek to zero only when needed
 210     if(output.tellg() > start) output.seekg(start);
 211     // and just in case
 212     output.clear();
 213   }
 214   // now go over every line and publish it to the ResultsChecker
 215   std::string line;
 216   bool on_first = true;
 217   while (output.eof() == false) {
 218     CHECK(output.good());
 219     std::getline(output, line);
 220     if (on_first) {
 221       SetHeader_(line); // this is important
 222       on_first = false;
 223       continue;
 224     }
 225     SetValues_(line);
 226   }
 227   // finally we can call the subscribed check functions
 228   for(const auto& p : check_patterns) {
 229     VLOG(2) << "--------------------------------\n";
 230     VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
 231     for(const auto& r : results) {
 232       if(!p.regex->Match(r.name)) {
 233         VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
 234         continue;
 235       } else {
 236         VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
 237       }
 238       VLOG(1) << "Checking results of " << r.name << ": ... \n";
 239       p.fn(r);
 240       VLOG(1) << "Checking results of " << r.name << ": OK.\n";
 241     }
 242   }
 243 }
 244
 245 // prepare for the names in this header
 246 void ResultsChecker::SetHeader_(const std::string& csv_header) {
 247   field_names = SplitCsv_(csv_header);
 248 }
 249
 250 // set the values for a benchmark
 251 void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
 252   if(entry_csv_line.empty()) return; // some lines are empty
 253   CHECK(!field_names.empty());
 254   auto vals = SplitCsv_(entry_csv_line);
 255   CHECK_EQ(vals.size(), field_names.size());
 256   results.emplace_back(vals[0]); // vals[0] is the benchmark name
 257   auto &entry = results.back();
 258   for (size_t i = 1, e = vals.size(); i < e; ++i) {
 259     entry.values[field_names[i]] = vals[i];
 260   }
 261 }
 262
 263 // a quick'n'dirty csv splitter (eliminating quotes)
 264 std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) {
 265   std::vector< std::string > out;
 266   if(line.empty()) return out;
 267   if(!field_names.empty()) out.reserve(field_names.size());
 268   size_t prev = 0, pos = line.find_first_of(','), curr = pos;
 269   while(pos != line.npos) {
 270     CHECK(curr > 0);
 271     if(line[prev] == '"') ++prev;
 272     if(line[curr-1] == '"') --curr;
 273     out.push_back(line.substr(prev, curr-prev));
 274     prev = pos + 1;
 275     pos = line.find_first_of(',', pos + 1);
 276     curr = pos;
 277   }
 278   curr = line.size();
 279   if(line[prev] == '"') ++prev;
 280   if(line[curr-1] == '"') --curr;
 281   out.push_back(line.substr(prev, curr-prev));
 282   return out;
 283 }
 284
 285 }  // end namespace internal
 286
 287 size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
 288 {
 289   auto &rc = internal::GetResultsChecker();
 290   rc.Add(bm_name, fn);
 291   return rc.results.size();
 292 }
 293
 294 int Results::NumThreads() const {
 295   auto pos = name.find("/threads:");
 296   if(pos == name.npos) return 1;
 297   auto end = name.find('/', pos + 9);
 298   std::stringstream ss;
 299   ss << name.substr(pos + 9, end);
 300   int num = 1;
 301   ss >> num;
 302   CHECK(!ss.fail());
 303   return num;
 304 }
 305
 306 double Results::GetTime(BenchmarkTime which) const {
 307   CHECK(which == kCpuTime || which == kRealTime);
 308   const char *which_str = which == kCpuTime ? "cpu_time" : "real_time";
 309   double val = GetAs< double >(which_str);
 310   auto unit = Get("time_unit");
 311   CHECK(unit);
 312   if(*unit == "ns") {
 313     return val * 1.e-9;
 314   } else if(*unit == "us") {
 315     return val * 1.e-6;
 316   } else if(*unit == "ms") {
 317     return val * 1.e-3;
 318   } else if(*unit == "s") {
 319     return val;
 320   } else {
 321     CHECK(1 == 0) << "unknown time unit: " << *unit;
 322     return 0;
 323   }
 324 }
 325
 326 // ========================================================================= //
 327 // -------------------------- Public API Definitions------------------------ //
 328 // ========================================================================= //
 329
 330 TestCase::TestCase(std::string re, int rule)
 331     : regex_str(std::move(re)),
 332       match_rule(rule),
 333       substituted_regex(internal::PerformSubstitutions(regex_str)),
 334       regex(std::make_shared<benchmark::Regex>()) {
 335   std::string err_str;
 336   regex->Init(substituted_regex,& err_str);
 337   CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
 338                          << "\""
 339                          << "\n    originally \"" << regex_str << "\""
 340                          << "\n    got error: " << err_str;
 341 }
 342
 343 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
 344   auto& L = internal::GetTestCaseList(ID);
 345   L.insert(L.end(), il);
 346   return 0;
 347 }
 348
 349 int SetSubstitutions(
 350     std::initializer_list<std::pair<std::string, std::string>> il) {
 351   auto& subs = internal::GetSubstitutions();
 352   for (auto KV : il) {
 353     bool exists = false;
 354     KV.second = internal::PerformSubstitutions(KV.second);
 355     for (auto& EKV : subs) {
 356       if (EKV.first == KV.first) {
 357         EKV.second = std::move(KV.second);
 358         exists = true;
 359         break;
 360       }
 361     }
 362     if (!exists) subs.push_back(std::move(KV));
 363   }
 364   return 0;
 365 }
 366
 367 void RunOutputTests(int argc, char* argv[]) {
 368   using internal::GetTestCaseList;
 369   benchmark::Initialize(&argc, argv);
 370   auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true);
 371   benchmark::ConsoleReporter CR(options);
 372   benchmark::JSONReporter JR;
 373   benchmark::CSVReporter CSVR;
 374   struct ReporterTest {
 375     const char* name;
 376     std::vector<TestCase>& output_cases;
 377     std::vector<TestCase>& error_cases;
 378     benchmark::BenchmarkReporter& reporter;
 379     std::stringstream out_stream;
 380     std::stringstream err_stream;
 381
 382     ReporterTest(const char* n, std::vector<TestCase>& out_tc,
 383                  std::vector<TestCase>& err_tc,
 384                  benchmark::BenchmarkReporter& br)
 385         : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
 386       reporter.SetOutputStream(&out_stream);
 387       reporter.SetErrorStream(&err_stream);
 388     }
 389   } TestCases[] = {
 390       {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
 391        GetTestCaseList(TC_ConsoleErr), CR},
 392       {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
 393        JR},
 394       {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
 395        CSVR},
 396   };
 397
 398   // Create the test reporter and run the benchmarks.
 399   std::cout << "Running benchmarks...\n";
 400   internal::TestReporter test_rep({&CR, &JR, &CSVR});
 401   benchmark::RunSpecifiedBenchmarks(&test_rep);
 402
 403   for (auto& rep_test : TestCases) {
 404     std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
 405     std::string banner(msg.size() - 1, '-');
 406     std::cout << banner << msg << banner << "\n";
 407
 408     std::cerr << rep_test.err_stream.str();
 409     std::cout << rep_test.out_stream.str();
 410
 411     internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
 412     internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
 413
 414     std::cout << "\n";
 415   }
 416
 417   // now that we know the output is as expected, we can dispatch
 418   // the checks to subscribees.
 419   auto &csv = TestCases[2];
 420   // would use == but gcc spits a warning
 421   CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
 422   internal::GetResultsChecker().CheckResults(csv.out_stream);
 423 }