utils/benchmark/test/user_counters_test.cc

   1
   2 #undef NDEBUG
   3
   4 #include "benchmark/benchmark.h"
   5 #include "output_test.h"
   6
   7 // ========================================================================= //
   8 // ---------------------- Testing Prologue Output -------------------------- //
   9 // ========================================================================= //
  10
  11 ADD_CASES(TC_ConsoleOut,
  12           {{"^[-]+$", MR_Next},
  13            {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next},
  14            {"^[-]+$", MR_Next}});
  15 ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}});
  16
  17 // ========================================================================= //
  18 // ------------------------- Simple Counters Output ------------------------ //
  19 // ========================================================================= //
  20
  21 void BM_Counters_Simple(benchmark::State& state) {
  22   for (auto _ : state) {
  23   }
  24   state.counters["foo"] = 1;
  25   state.counters["bar"] = 2 * (double)state.iterations();
  26 }
  27 BENCHMARK(BM_Counters_Simple);
  28 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
  29 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
  30                        {"\"iterations\": %int,$", MR_Next},
  31                        {"\"real_time\": %float,$", MR_Next},
  32                        {"\"cpu_time\": %float,$", MR_Next},
  33                        {"\"time_unit\": \"ns\",$", MR_Next},
  34                        {"\"bar\": %float,$", MR_Next},
  35                        {"\"foo\": %float$", MR_Next},
  36                        {"}", MR_Next}});
  37 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
  38 // VS2013 does not allow this function to be passed as a lambda argument
  39 // to CHECK_BENCHMARK_RESULTS()
  40 void CheckSimple(Results const& e) {
  41   double its = e.GetAs< double >("iterations");
  42   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
  43   // check that the value of bar is within 0.1% of the expected value
  44   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2.*its, 0.001);
  45 }
  46 CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple);
  47
  48 // ========================================================================= //
  49 // --------------------- Counters+Items+Bytes/s Output --------------------- //
  50 // ========================================================================= //
  51
  52 namespace { int num_calls1 = 0; }
  53 void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
  54   for (auto _ : state) {
  55   }
  56   state.counters["foo"] = 1;
  57   state.counters["bar"] = ++num_calls1;
  58   state.SetBytesProcessed(364);
  59   state.SetItemsProcessed(150);
  60 }
  61 BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
  62 ADD_CASES(TC_ConsoleOut,
  63           {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
  64             "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
  65 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
  66                        {"\"iterations\": %int,$", MR_Next},
  67                        {"\"real_time\": %float,$", MR_Next},
  68                        {"\"cpu_time\": %float,$", MR_Next},
  69                        {"\"time_unit\": \"ns\",$", MR_Next},
  70                        {"\"bytes_per_second\": %float,$", MR_Next},
  71                        {"\"items_per_second\": %float,$", MR_Next},
  72                        {"\"bar\": %float,$", MR_Next},
  73                        {"\"foo\": %float$", MR_Next},
  74                        {"}", MR_Next}});
  75 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
  76                        "%csv_bytes_items_report,%float,%float$"}});
  77 // VS2013 does not allow this function to be passed as a lambda argument
  78 // to CHECK_BENCHMARK_RESULTS()
  79 void CheckBytesAndItemsPSec(Results const& e) {
  80   double t = e.DurationCPUTime(); // this (and not real time) is the time used
  81   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
  82   CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
  83   // check that the values are within 0.1% of the expected values
  84   CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364./t, 0.001);
  85   CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150./t, 0.001);
  86 }
  87 CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
  88                         &CheckBytesAndItemsPSec);
  89
  90 // ========================================================================= //
  91 // ------------------------- Rate Counters Output -------------------------- //
  92 // ========================================================================= //
  93
  94 void BM_Counters_Rate(benchmark::State& state) {
  95   for (auto _ : state) {
  96   }
  97   namespace bm = benchmark;
  98   state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
  99   state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
 100 }
 101 BENCHMARK(BM_Counters_Rate);
 102 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
 103 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
 104                        {"\"iterations\": %int,$", MR_Next},
 105                        {"\"real_time\": %float,$", MR_Next},
 106                        {"\"cpu_time\": %float,$", MR_Next},
 107                        {"\"time_unit\": \"ns\",$", MR_Next},
 108                        {"\"bar\": %float,$", MR_Next},
 109                        {"\"foo\": %float$", MR_Next},
 110                        {"}", MR_Next}});
 111 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}});
 112 // VS2013 does not allow this function to be passed as a lambda argument
 113 // to CHECK_BENCHMARK_RESULTS()
 114 void CheckRate(Results const& e) {
 115   double t = e.DurationCPUTime(); // this (and not real time) is the time used
 116   // check that the values are within 0.1% of the expected values
 117   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./t, 0.001);
 118   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./t, 0.001);
 119 }
 120 CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate);
 121
 122 // ========================================================================= //
 123 // ------------------------- Thread Counters Output ------------------------ //
 124 // ========================================================================= //
 125
 126 void BM_Counters_Threads(benchmark::State& state) {
 127   for (auto _ : state) {
 128   }
 129   state.counters["foo"] = 1;
 130   state.counters["bar"] = 2;
 131 }
 132 BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
 133 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
 134 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
 135                        {"\"iterations\": %int,$", MR_Next},
 136                        {"\"real_time\": %float,$", MR_Next},
 137                        {"\"cpu_time\": %float,$", MR_Next},
 138                        {"\"time_unit\": \"ns\",$", MR_Next},
 139                        {"\"bar\": %float,$", MR_Next},
 140                        {"\"foo\": %float$", MR_Next},
 141                        {"}", MR_Next}});
 142 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}});
 143 // VS2013 does not allow this function to be passed as a lambda argument
 144 // to CHECK_BENCHMARK_RESULTS()
 145 void CheckThreads(Results const& e) {
 146   CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads());
 147   CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads());
 148 }
 149 CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads);
 150
 151 // ========================================================================= //
 152 // ---------------------- ThreadAvg Counters Output ------------------------ //
 153 // ========================================================================= //
 154
 155 void BM_Counters_AvgThreads(benchmark::State& state) {
 156   for (auto _ : state) {
 157   }
 158   namespace bm = benchmark;
 159   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads};
 160   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
 161 }
 162 BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
 163 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
 164 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
 165                        {"\"iterations\": %int,$", MR_Next},
 166                        {"\"real_time\": %float,$", MR_Next},
 167                        {"\"cpu_time\": %float,$", MR_Next},
 168                        {"\"time_unit\": \"ns\",$", MR_Next},
 169                        {"\"bar\": %float,$", MR_Next},
 170                        {"\"foo\": %float$", MR_Next},
 171                        {"}", MR_Next}});
 172 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}});
 173 // VS2013 does not allow this function to be passed as a lambda argument
 174 // to CHECK_BENCHMARK_RESULTS()
 175 void CheckAvgThreads(Results const& e) {
 176   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
 177   CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
 178 }
 179 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int",
 180                         &CheckAvgThreads);
 181
 182 // ========================================================================= //
 183 // ---------------------- ThreadAvg Counters Output ------------------------ //
 184 // ========================================================================= //
 185
 186 void BM_Counters_AvgThreadsRate(benchmark::State& state) {
 187   for (auto _ : state) {
 188   }
 189   namespace bm = benchmark;
 190   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};
 191   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
 192 }
 193 BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
 194 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
 195 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
 196                        {"\"iterations\": %int,$", MR_Next},
 197                        {"\"real_time\": %float,$", MR_Next},
 198                        {"\"cpu_time\": %float,$", MR_Next},
 199                        {"\"time_unit\": \"ns\",$", MR_Next},
 200                        {"\"bar\": %float,$", MR_Next},
 201                        {"\"foo\": %float$", MR_Next},
 202                        {"}", MR_Next}});
 203 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/threads:%int\",%csv_report,%float,%float$"}});
 204 // VS2013 does not allow this function to be passed as a lambda argument
 205 // to CHECK_BENCHMARK_RESULTS()
 206 void CheckAvgThreadsRate(Results const& e) {
 207   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./e.DurationCPUTime(), 0.001);
 208   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./e.DurationCPUTime(), 0.001);
 209 }
 210 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int",
 211                         &CheckAvgThreadsRate);
 212
 213 // ========================================================================= //
 214 // --------------------------- TEST CASES END ------------------------------ //
 215 // ========================================================================= //
 216
 217 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }