utils/benchmark/test/user_counters_tabular_test.cc

   1
   2 #undef NDEBUG
   3
   4 #include "benchmark/benchmark.h"
   5 #include "output_test.h"
   6
   7 // @todo: <jpmag> this checks the full output at once; the rule for
   8 // CounterSet1 was failing because it was not matching "^[-]+$".
   9 // @todo: <jpmag> check that the counters are vertically aligned.
  10 ADD_CASES(TC_ConsoleOut, {
  11 // keeping these lines long improves readability, so:
  12 // clang-format off
  13     {"^[-]+$", MR_Next},
  14     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
  15     {"^[-]+$", MR_Next},
  16     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  17     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  18     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  19     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  20     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  21     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  22     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  23     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  24     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  25     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  26     {"^[-]+$", MR_Next},
  27     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
  28     {"^[-]+$", MR_Next},
  29     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  30     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  31     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  32     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  33     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  34     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  35     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  36     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  37     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  38     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  39     {"^[-]+$", MR_Next},
  40     {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
  41     {"^[-]+$", MR_Next},
  42     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  43     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  44     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  45     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  46     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
  47 // clang-format on
  48 });
  49 ADD_CASES(TC_CSVOut, {{"%csv_header,"
  50                        "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
  51
  52 // ========================================================================= //
  53 // ------------------------- Tabular Counters Output ----------------------- //
  54 // ========================================================================= //
  55
  56 void BM_Counters_Tabular(benchmark::State& state) {
  57   for (auto _ : state) {
  58   }
  59   namespace bm = benchmark;
  60   state.counters.insert({
  61     {"Foo",  { 1, bm::Counter::kAvgThreads}},
  62     {"Bar",  { 2, bm::Counter::kAvgThreads}},
  63     {"Baz",  { 4, bm::Counter::kAvgThreads}},
  64     {"Bat",  { 8, bm::Counter::kAvgThreads}},
  65     {"Frob", {16, bm::Counter::kAvgThreads}},
  66     {"Lob",  {32, bm::Counter::kAvgThreads}},
  67   });
  68 }
  69 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
  70 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
  71                        {"\"iterations\": %int,$", MR_Next},
  72                        {"\"real_time\": %float,$", MR_Next},
  73                        {"\"cpu_time\": %float,$", MR_Next},
  74                        {"\"time_unit\": \"ns\",$", MR_Next},
  75                        {"\"Bar\": %float,$", MR_Next},
  76                        {"\"Bat\": %float,$", MR_Next},
  77                        {"\"Baz\": %float,$", MR_Next},
  78                        {"\"Foo\": %float,$", MR_Next},
  79                        {"\"Frob\": %float,$", MR_Next},
  80                        {"\"Lob\": %float$", MR_Next},
  81                        {"}", MR_Next}});
  82 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
  83                        "%float,%float,%float,%float,%float,%float$"}});
  84 // VS2013 does not allow this function to be passed as a lambda argument
  85 // to CHECK_BENCHMARK_RESULTS()
  86 void CheckTabular(Results const& e) {
  87   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
  88   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
  89   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
  90   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
  91   CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
  92   CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
  93 }
  94 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
  95
  96 // ========================================================================= //
  97 // -------------------- Tabular+Rate Counters Output ----------------------- //
  98 // ========================================================================= //
  99
 100 void BM_CounterRates_Tabular(benchmark::State& state) {
 101   for (auto _ : state) {
 102   }
 103   namespace bm = benchmark;
 104   state.counters.insert({
 105     {"Foo",  { 1, bm::Counter::kAvgThreadsRate}},
 106     {"Bar",  { 2, bm::Counter::kAvgThreadsRate}},
 107     {"Baz",  { 4, bm::Counter::kAvgThreadsRate}},
 108     {"Bat",  { 8, bm::Counter::kAvgThreadsRate}},
 109     {"Frob", {16, bm::Counter::kAvgThreadsRate}},
 110     {"Lob",  {32, bm::Counter::kAvgThreadsRate}},
 111   });
 112 }
 113 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
 114 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
 115                        {"\"iterations\": %int,$", MR_Next},
 116                        {"\"real_time\": %float,$", MR_Next},
 117                        {"\"cpu_time\": %float,$", MR_Next},
 118                        {"\"time_unit\": \"ns\",$", MR_Next},
 119                        {"\"Bar\": %float,$", MR_Next},
 120                        {"\"Bat\": %float,$", MR_Next},
 121                        {"\"Baz\": %float,$", MR_Next},
 122                        {"\"Foo\": %float,$", MR_Next},
 123                        {"\"Frob\": %float,$", MR_Next},
 124                        {"\"Lob\": %float$", MR_Next},
 125                        {"}", MR_Next}});
 126 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
 127                        "%float,%float,%float,%float,%float,%float$"}});
 128 // VS2013 does not allow this function to be passed as a lambda argument
 129 // to CHECK_BENCHMARK_RESULTS()
 130 void CheckTabularRate(Results const& e) {
 131   double t = e.DurationCPUTime();
 132   CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001);
 133   CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001);
 134   CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001);
 135   CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001);
 136   CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001);
 137   CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001);
 138 }
 139 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
 140                         &CheckTabularRate);
 141
 142 // ========================================================================= //
 143 // ------------------------- Tabular Counters Output ----------------------- //
 144 // ========================================================================= //
 145
 146 // set only some of the counters
 147 void BM_CounterSet0_Tabular(benchmark::State& state) {
 148   for (auto _ : state) {
 149   }
 150   namespace bm = benchmark;
 151   state.counters.insert({
 152     {"Foo", {10, bm::Counter::kAvgThreads}},
 153     {"Bar", {20, bm::Counter::kAvgThreads}},
 154     {"Baz", {40, bm::Counter::kAvgThreads}},
 155   });
 156 }
 157 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
 158 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
 159                        {"\"iterations\": %int,$", MR_Next},
 160                        {"\"real_time\": %float,$", MR_Next},
 161                        {"\"cpu_time\": %float,$", MR_Next},
 162                        {"\"time_unit\": \"ns\",$", MR_Next},
 163                        {"\"Bar\": %float,$", MR_Next},
 164                        {"\"Baz\": %float,$", MR_Next},
 165                        {"\"Foo\": %float$", MR_Next},
 166                        {"}", MR_Next}});
 167 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
 168                        "%float,,%float,%float,,"}});
 169 // VS2013 does not allow this function to be passed as a lambda argument
 170 // to CHECK_BENCHMARK_RESULTS()
 171 void CheckSet0(Results const& e) {
 172   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 173   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
 174   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 175 }
 176 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
 177
 178 // again.
 179 void BM_CounterSet1_Tabular(benchmark::State& state) {
 180   for (auto _ : state) {
 181   }
 182   namespace bm = benchmark;
 183   state.counters.insert({
 184     {"Foo", {15, bm::Counter::kAvgThreads}},
 185     {"Bar", {25, bm::Counter::kAvgThreads}},
 186     {"Baz", {45, bm::Counter::kAvgThreads}},
 187   });
 188 }
 189 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
 190 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
 191                        {"\"iterations\": %int,$", MR_Next},
 192                        {"\"real_time\": %float,$", MR_Next},
 193                        {"\"cpu_time\": %float,$", MR_Next},
 194                        {"\"time_unit\": \"ns\",$", MR_Next},
 195                        {"\"Bar\": %float,$", MR_Next},
 196                        {"\"Baz\": %float,$", MR_Next},
 197                        {"\"Foo\": %float$", MR_Next},
 198                        {"}", MR_Next}});
 199 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
 200                        "%float,,%float,%float,,"}});
 201 // VS2013 does not allow this function to be passed as a lambda argument
 202 // to CHECK_BENCHMARK_RESULTS()
 203 void CheckSet1(Results const& e) {
 204   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
 205   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
 206   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
 207 }
 208 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
 209
 210 // ========================================================================= //
 211 // ------------------------- Tabular Counters Output ----------------------- //
 212 // ========================================================================= //
 213
 214 // set only some of the counters, different set now.
 215 void BM_CounterSet2_Tabular(benchmark::State& state) {
 216   for (auto _ : state) {
 217   }
 218   namespace bm = benchmark;
 219   state.counters.insert({
 220     {"Foo", {10, bm::Counter::kAvgThreads}},
 221     {"Bat", {30, bm::Counter::kAvgThreads}},
 222     {"Baz", {40, bm::Counter::kAvgThreads}},
 223   });
 224 }
 225 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
 226 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
 227                        {"\"iterations\": %int,$", MR_Next},
 228                        {"\"real_time\": %float,$", MR_Next},
 229                        {"\"cpu_time\": %float,$", MR_Next},
 230                        {"\"time_unit\": \"ns\",$", MR_Next},
 231                        {"\"Bat\": %float,$", MR_Next},
 232                        {"\"Baz\": %float,$", MR_Next},
 233                        {"\"Foo\": %float$", MR_Next},
 234                        {"}", MR_Next}});
 235 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
 236                        ",%float,%float,%float,,"}});
 237 // VS2013 does not allow this function to be passed as a lambda argument
 238 // to CHECK_BENCHMARK_RESULTS()
 239 void CheckSet2(Results const& e) {
 240   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 241   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
 242   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 243 }
 244 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
 245
 246 // ========================================================================= //
 247 // --------------------------- TEST CASES END ------------------------------ //
 248 // ========================================================================= //
 249
 250 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }