utils/benchmark/test/register_benchmark_test.cc

   1
   2 #undef NDEBUG
   3 #include <cassert>
   4 #include <vector>
   5
   6 #include "../src/check.h"  // NOTE: check.h is for internal use only!
   7 #include "benchmark/benchmark.h"
   8
   9 namespace {
  10
  11 class TestReporter : public benchmark::ConsoleReporter {
  12  public:
  13   virtual void ReportRuns(const std::vector<Run>& report) {
  14     all_runs_.insert(all_runs_.end(), begin(report), end(report));
  15     ConsoleReporter::ReportRuns(report);
  16   }
  17
  18   std::vector<Run> all_runs_;
  19 };
  20
  21 struct TestCase {
  22   std::string name;
  23   const char* label;
  24   // Note: not explicit as we rely on it being converted through ADD_CASES.
  25   TestCase(const char* xname) : TestCase(xname, nullptr) {}
  26   TestCase(const char* xname, const char* xlabel)
  27       : name(xname), label(xlabel) {}
  28
  29   typedef benchmark::BenchmarkReporter::Run Run;
  30
  31   void CheckRun(Run const& run) const {
  32     CHECK(name == run.benchmark_name) << "expected " << name << " got "
  33                                       << run.benchmark_name;
  34     if (label) {
  35       CHECK(run.report_label == label) << "expected " << label << " got "
  36                                        << run.report_label;
  37     } else {
  38       CHECK(run.report_label == "");
  39     }
  40   }
  41 };
  42
  43 std::vector<TestCase> ExpectedResults;
  44
  45 int AddCases(std::initializer_list<TestCase> const& v) {
  46   for (auto N : v) {
  47     ExpectedResults.push_back(N);
  48   }
  49   return 0;
  50 }
  51
  52 #define CONCAT(x, y) CONCAT2(x, y)
  53 #define CONCAT2(x, y) x##y
  54 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__})
  55
  56 }  // end namespace
  57
  58 typedef benchmark::internal::Benchmark* ReturnVal;
  59
  60 //----------------------------------------------------------------------------//
  61 // Test RegisterBenchmark with no additional arguments
  62 //----------------------------------------------------------------------------//
  63 void BM_function(benchmark::State& state) {
  64   for (auto _ : state) {
  65   }
  66 }
  67 BENCHMARK(BM_function);
  68 ReturnVal dummy = benchmark::RegisterBenchmark(
  69     "BM_function_manual_registration", BM_function);
  70 ADD_CASES({"BM_function"}, {"BM_function_manual_registration"});
  71
  72 //----------------------------------------------------------------------------//
  73 // Test RegisterBenchmark with additional arguments
  74 // Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they
  75 //       reject the variadic pack expansion of lambda captures.
  76 //----------------------------------------------------------------------------//
  77 #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK
  78
  79 void BM_extra_args(benchmark::State& st, const char* label) {
  80   for (auto _ : st) {
  81   }
  82   st.SetLabel(label);
  83 }
  84 int RegisterFromFunction() {
  85   std::pair<const char*, const char*> cases[] = {
  86       {"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}};
  87   for (auto const& c : cases)
  88     benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second);
  89   return 0;
  90 }
  91 int dummy2 = RegisterFromFunction();
  92 ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"});
  93
  94 #endif  // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK
  95
  96 //----------------------------------------------------------------------------//
  97 // Test RegisterBenchmark with different callable types
  98 //----------------------------------------------------------------------------//
  99
 100 struct CustomFixture {
 101   void operator()(benchmark::State& st) {
 102     for (auto _ : st) {
 103     }
 104   }
 105 };
 106
 107 void TestRegistrationAtRuntime() {
 108 #ifdef BENCHMARK_HAS_CXX11
 109   {
 110     CustomFixture fx;
 111     benchmark::RegisterBenchmark("custom_fixture", fx);
 112     AddCases({"custom_fixture"});
 113   }
 114 #endif
 115 #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK
 116   {
 117     const char* x = "42";
 118     auto capturing_lam = [=](benchmark::State& st) {
 119       for (auto _ : st) {
 120       }
 121       st.SetLabel(x);
 122     };
 123     benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam);
 124     AddCases({{"lambda_benchmark", x}});
 125   }
 126 #endif
 127 }
 128
 129 // Test that all benchmarks, registered at either during static init or runtime,
 130 // are run and the results are passed to the reported.
 131 void RunTestOne() {
 132   TestRegistrationAtRuntime();
 133
 134   TestReporter test_reporter;
 135   benchmark::RunSpecifiedBenchmarks(&test_reporter);
 136
 137   typedef benchmark::BenchmarkReporter::Run Run;
 138   auto EB = ExpectedResults.begin();
 139
 140   for (Run const& run : test_reporter.all_runs_) {
 141     assert(EB != ExpectedResults.end());
 142     EB->CheckRun(run);
 143     ++EB;
 144   }
 145   assert(EB == ExpectedResults.end());
 146 }
 147
 148 // Test that ClearRegisteredBenchmarks() clears all previously registered
 149 // benchmarks.
 150 // Also test that new benchmarks can be registered and ran afterwards.
 151 void RunTestTwo() {
 152   assert(ExpectedResults.size() != 0 &&
 153          "must have at least one registered benchmark");
 154   ExpectedResults.clear();
 155   benchmark::ClearRegisteredBenchmarks();
 156
 157   TestReporter test_reporter;
 158   size_t num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter);
 159   assert(num_ran == 0);
 160   assert(test_reporter.all_runs_.begin() == test_reporter.all_runs_.end());
 161
 162   TestRegistrationAtRuntime();
 163   num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter);
 164   assert(num_ran == ExpectedResults.size());
 165
 166   typedef benchmark::BenchmarkReporter::Run Run;
 167   auto EB = ExpectedResults.begin();
 168
 169   for (Run const& run : test_reporter.all_runs_) {
 170     assert(EB != ExpectedResults.end());
 171     EB->CheckRun(run);
 172     ++EB;
 173   }
 174   assert(EB == ExpectedResults.end());
 175 }
 176
 177 int main(int argc, char* argv[]) {
 178   benchmark::Initialize(&argc, argv);
 179
 180   RunTestOne();
 181   RunTestTwo();
 182 }