third-party/benchmark/test/skip_with_error_test.cc

   1
   2 #undef NDEBUG
   3 #include <cassert>
   4 #include <vector>
   5
   6 #include "../src/check.h"  // NOTE: check.h is for internal use only!
   7 #include "benchmark/benchmark.h"
   8
   9 namespace {
  10
  11 class TestReporter : public benchmark::ConsoleReporter {
  12  public:
  13   virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
  14     return ConsoleReporter::ReportContext(context);
  15   };
  16
  17   virtual void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
  18     all_runs_.insert(all_runs_.end(), begin(report), end(report));
  19     ConsoleReporter::ReportRuns(report);
  20   }
  21
  22   TestReporter() {}
  23   virtual ~TestReporter() {}
  24
  25   mutable std::vector<Run> all_runs_;
  26 };
  27
  28 struct TestCase {
  29   std::string name;
  30   bool error_occurred;
  31   std::string error_message;
  32
  33   typedef benchmark::BenchmarkReporter::Run Run;
  34
  35   void CheckRun(Run const& run) const {
  36     BM_CHECK(name == run.benchmark_name())
  37         << "expected " << name << " got " << run.benchmark_name();
  38     BM_CHECK(error_occurred == run.error_occurred);
  39     BM_CHECK(error_message == run.error_message);
  40     if (error_occurred) {
  41       // BM_CHECK(run.iterations == 0);
  42     } else {
  43       BM_CHECK(run.iterations != 0);
  44     }
  45   }
  46 };
  47
  48 std::vector<TestCase> ExpectedResults;
  49
  50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
  51   for (auto TC : v) {
  52     TC.name = base_name + TC.name;
  53     ExpectedResults.push_back(std::move(TC));
  54   }
  55   return 0;
  56 }
  57
  58 #define CONCAT(x, y) CONCAT2(x, y)
  59 #define CONCAT2(x, y) x##y
  60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
  61
  62 }  // end namespace
  63
  64 void BM_error_no_running(benchmark::State& state) {
  65   state.SkipWithError("error message");
  66 }
  67 BENCHMARK(BM_error_no_running);
  68 ADD_CASES("BM_error_no_running", {{"", true, "error message"}});
  69
  70 void BM_error_before_running(benchmark::State& state) {
  71   state.SkipWithError("error message");
  72   while (state.KeepRunning()) {
  73     assert(false);
  74   }
  75 }
  76 BENCHMARK(BM_error_before_running);
  77 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
  78
  79 void BM_error_before_running_batch(benchmark::State& state) {
  80   state.SkipWithError("error message");
  81   while (state.KeepRunningBatch(17)) {
  82     assert(false);
  83   }
  84 }
  85 BENCHMARK(BM_error_before_running_batch);
  86 ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});
  87
  88 void BM_error_before_running_range_for(benchmark::State& state) {
  89   state.SkipWithError("error message");
  90   for (auto _ : state) {
  91     assert(false);
  92   }
  93 }
  94 BENCHMARK(BM_error_before_running_range_for);
  95 ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
  96
  97 void BM_error_during_running(benchmark::State& state) {
  98   int first_iter = true;
  99   while (state.KeepRunning()) {
 100     if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
 101       assert(first_iter);
 102       first_iter = false;
 103       state.SkipWithError("error message");
 104     } else {
 105       state.PauseTiming();
 106       state.ResumeTiming();
 107     }
 108   }
 109 }
 110 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
 111 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
 112                                       {"/1/threads:2", true, "error message"},
 113                                       {"/1/threads:4", true, "error message"},
 114                                       {"/1/threads:8", true, "error message"},
 115                                       {"/2/threads:1", false, ""},
 116                                       {"/2/threads:2", false, ""},
 117                                       {"/2/threads:4", false, ""},
 118                                       {"/2/threads:8", false, ""}});
 119
 120 void BM_error_during_running_ranged_for(benchmark::State& state) {
 121   assert(state.max_iterations > 3 && "test requires at least a few iterations");
 122   bool first_iter = true;
 123   // NOTE: Users should not write the for loop explicitly.
 124   for (auto It = state.begin(), End = state.end(); It != End; ++It) {
 125     if (state.range(0) == 1) {
 126       assert(first_iter);
 127       first_iter = false;
 128       (void)first_iter;
 129       state.SkipWithError("error message");
 130       // Test the unfortunate but documented behavior that the ranged-for loop
 131       // doesn't automatically terminate when SkipWithError is set.
 132       assert(++It != End);
 133       break;  // Required behavior
 134     }
 135   }
 136 }
 137 BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
 138 ADD_CASES("BM_error_during_running_ranged_for",
 139           {{"/1/iterations:5", true, "error message"},
 140            {"/2/iterations:5", false, ""}});
 141
 142 void BM_error_after_running(benchmark::State& state) {
 143   for (auto _ : state) {
 144     benchmark::DoNotOptimize(state.iterations());
 145   }
 146   if (state.thread_index() <= (state.threads() / 2))
 147     state.SkipWithError("error message");
 148 }
 149 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
 150 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
 151                                      {"/threads:2", true, "error message"},
 152                                      {"/threads:4", true, "error message"},
 153                                      {"/threads:8", true, "error message"}});
 154
 155 void BM_error_while_paused(benchmark::State& state) {
 156   bool first_iter = true;
 157   while (state.KeepRunning()) {
 158     if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
 159       assert(first_iter);
 160       first_iter = false;
 161       state.PauseTiming();
 162       state.SkipWithError("error message");
 163     } else {
 164       state.PauseTiming();
 165       state.ResumeTiming();
 166     }
 167   }
 168 }
 169 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
 170 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
 171                                     {"/1/threads:2", true, "error message"},
 172                                     {"/1/threads:4", true, "error message"},
 173                                     {"/1/threads:8", true, "error message"},
 174                                     {"/2/threads:1", false, ""},
 175                                     {"/2/threads:2", false, ""},
 176                                     {"/2/threads:4", false, ""},
 177                                     {"/2/threads:8", false, ""}});
 178
 179 int main(int argc, char* argv[]) {
 180   benchmark::Initialize(&argc, argv);
 181
 182   TestReporter test_reporter;
 183   benchmark::RunSpecifiedBenchmarks(&test_reporter);
 184
 185   typedef benchmark::BenchmarkReporter::Run Run;
 186   auto EB = ExpectedResults.begin();
 187
 188   for (Run const& run : test_reporter.all_runs_) {
 189     assert(EB != ExpectedResults.end());
 190     EB->CheckRun(run);
 191     ++EB;
 192   }
 193   assert(EB == ExpectedResults.end());
 194
 195   return 0;
 196 }