third-party/benchmark/test/user_counters_tabular_test.cc

   1
   2 #undef NDEBUG
   3
   4 #include "benchmark/benchmark.h"
   5 #include "output_test.h"
   6
   7 // @todo: <jpmag> this checks the full output at once; the rule for
   8 // CounterSet1 was failing because it was not matching "^[-]+$".
   9 // @todo: <jpmag> check that the counters are vertically aligned.
  10 ADD_CASES(TC_ConsoleOut,
  11           {
  12               // keeping these lines long improves readability, so:
  13               // clang-format off
  14     {"^[-]+$", MR_Next},
  15     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
  16     {"^[-]+$", MR_Next},
  17       {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  18       {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  19       {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  20       {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  21             {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  22             {"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
  23       {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  24       {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  25       {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  26       {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  27             {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  28             {"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
  29     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  30     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  31     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  32     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  33     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  34     {"^[-]+$", MR_Next},
  35     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
  36     {"^[-]+$", MR_Next},
  37     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  38     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  39     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  40     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  41     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  42     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  43     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  44     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  45     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  46     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  47     {"^[-]+$", MR_Next},
  48     {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
  49     {"^[-]+$", MR_Next},
  50     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  51     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  52     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  53     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  54     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
  55               // clang-format on
  56           });
  57 ADD_CASES(TC_CSVOut, {{"%csv_header,"
  58                        "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
  59
  60 // ========================================================================= //
  61 // ------------------------- Tabular Counters Output ----------------------- //
  62 // ========================================================================= //
  63
  64 void BM_Counters_Tabular(benchmark::State& state) {
  65   for (auto _ : state) {
  66   }
  67   namespace bm = benchmark;
  68   state.counters.insert({
  69       {"Foo", {1, bm::Counter::kAvgThreads}},
  70       {"Bar", {2, bm::Counter::kAvgThreads}},
  71       {"Baz", {4, bm::Counter::kAvgThreads}},
  72       {"Bat", {8, bm::Counter::kAvgThreads}},
  73       {"Frob", {16, bm::Counter::kAvgThreads}},
  74       {"Lob", {32, bm::Counter::kAvgThreads}},
  75   });
  76 }
  77 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);
  78 ADD_CASES(TC_JSONOut,
  79           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
  80            {"\"family_index\": 0,$", MR_Next},
  81            {"\"per_family_instance_index\": 0,$", MR_Next},
  82            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
  83             MR_Next},
  84            {"\"run_type\": \"iteration\",$", MR_Next},
  85            {"\"repetitions\": 2,$", MR_Next},
  86            {"\"repetition_index\": 0,$", MR_Next},
  87            {"\"threads\": 1,$", MR_Next},
  88            {"\"iterations\": %int,$", MR_Next},
  89            {"\"real_time\": %float,$", MR_Next},
  90            {"\"cpu_time\": %float,$", MR_Next},
  91            {"\"time_unit\": \"ns\",$", MR_Next},
  92            {"\"Bar\": %float,$", MR_Next},
  93            {"\"Bat\": %float,$", MR_Next},
  94            {"\"Baz\": %float,$", MR_Next},
  95            {"\"Foo\": %float,$", MR_Next},
  96            {"\"Frob\": %float,$", MR_Next},
  97            {"\"Lob\": %float$", MR_Next},
  98            {"}", MR_Next}});
  99 ADD_CASES(TC_JSONOut,
 100           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
 101            {"\"family_index\": 0,$", MR_Next},
 102            {"\"per_family_instance_index\": 0,$", MR_Next},
 103            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 104             MR_Next},
 105            {"\"run_type\": \"iteration\",$", MR_Next},
 106            {"\"repetitions\": 2,$", MR_Next},
 107            {"\"repetition_index\": 1,$", MR_Next},
 108            {"\"threads\": 1,$", MR_Next},
 109            {"\"iterations\": %int,$", MR_Next},
 110            {"\"real_time\": %float,$", MR_Next},
 111            {"\"cpu_time\": %float,$", MR_Next},
 112            {"\"time_unit\": \"ns\",$", MR_Next},
 113            {"\"Bar\": %float,$", MR_Next},
 114            {"\"Bat\": %float,$", MR_Next},
 115            {"\"Baz\": %float,$", MR_Next},
 116            {"\"Foo\": %float,$", MR_Next},
 117            {"\"Frob\": %float,$", MR_Next},
 118            {"\"Lob\": %float$", MR_Next},
 119            {"}", MR_Next}});
 120 ADD_CASES(TC_JSONOut,
 121           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"},
 122            {"\"family_index\": 0,$", MR_Next},
 123            {"\"per_family_instance_index\": 0,$", MR_Next},
 124            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 125             MR_Next},
 126            {"\"run_type\": \"aggregate\",$", MR_Next},
 127            {"\"repetitions\": 2,$", MR_Next},
 128            {"\"threads\": 1,$", MR_Next},
 129            {"\"aggregate_name\": \"mean\",$", MR_Next},
 130            {"\"aggregate_unit\": \"time\",$", MR_Next},
 131            {"\"iterations\": %int,$", MR_Next},
 132            {"\"real_time\": %float,$", MR_Next},
 133            {"\"cpu_time\": %float,$", MR_Next},
 134            {"\"time_unit\": \"ns\",$", MR_Next},
 135            {"\"Bar\": %float,$", MR_Next},
 136            {"\"Bat\": %float,$", MR_Next},
 137            {"\"Baz\": %float,$", MR_Next},
 138            {"\"Foo\": %float,$", MR_Next},
 139            {"\"Frob\": %float,$", MR_Next},
 140            {"\"Lob\": %float$", MR_Next},
 141            {"}", MR_Next}});
 142 ADD_CASES(TC_JSONOut,
 143           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
 144            {"\"family_index\": 0,$", MR_Next},
 145            {"\"per_family_instance_index\": 0,$", MR_Next},
 146            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 147             MR_Next},
 148            {"\"run_type\": \"aggregate\",$", MR_Next},
 149            {"\"repetitions\": 2,$", MR_Next},
 150            {"\"threads\": 1,$", MR_Next},
 151            {"\"aggregate_name\": \"median\",$", MR_Next},
 152            {"\"aggregate_unit\": \"time\",$", MR_Next},
 153            {"\"iterations\": %int,$", MR_Next},
 154            {"\"real_time\": %float,$", MR_Next},
 155            {"\"cpu_time\": %float,$", MR_Next},
 156            {"\"time_unit\": \"ns\",$", MR_Next},
 157            {"\"Bar\": %float,$", MR_Next},
 158            {"\"Bat\": %float,$", MR_Next},
 159            {"\"Baz\": %float,$", MR_Next},
 160            {"\"Foo\": %float,$", MR_Next},
 161            {"\"Frob\": %float,$", MR_Next},
 162            {"\"Lob\": %float$", MR_Next},
 163            {"}", MR_Next}});
 164 ADD_CASES(TC_JSONOut,
 165           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
 166            {"\"family_index\": 0,$", MR_Next},
 167            {"\"per_family_instance_index\": 0,$", MR_Next},
 168            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 169             MR_Next},
 170            {"\"run_type\": \"aggregate\",$", MR_Next},
 171            {"\"repetitions\": 2,$", MR_Next},
 172            {"\"threads\": 1,$", MR_Next},
 173            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 174            {"\"aggregate_unit\": \"time\",$", MR_Next},
 175            {"\"iterations\": %int,$", MR_Next},
 176            {"\"real_time\": %float,$", MR_Next},
 177            {"\"cpu_time\": %float,$", MR_Next},
 178            {"\"time_unit\": \"ns\",$", MR_Next},
 179            {"\"Bar\": %float,$", MR_Next},
 180            {"\"Bat\": %float,$", MR_Next},
 181            {"\"Baz\": %float,$", MR_Next},
 182            {"\"Foo\": %float,$", MR_Next},
 183            {"\"Frob\": %float,$", MR_Next},
 184            {"\"Lob\": %float$", MR_Next},
 185            {"}", MR_Next}});
 186 ADD_CASES(TC_JSONOut,
 187           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
 188            {"\"family_index\": 0,$", MR_Next},
 189            {"\"per_family_instance_index\": 0,$", MR_Next},
 190            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 191             MR_Next},
 192            {"\"run_type\": \"aggregate\",$", MR_Next},
 193            {"\"repetitions\": 2,$", MR_Next},
 194            {"\"threads\": 1,$", MR_Next},
 195            {"\"aggregate_name\": \"cv\",$", MR_Next},
 196            {"\"aggregate_unit\": \"percentage\",$", MR_Next},
 197            {"\"iterations\": %int,$", MR_Next},
 198            {"\"real_time\": %float,$", MR_Next},
 199            {"\"cpu_time\": %float,$", MR_Next},
 200            {"\"time_unit\": \"ns\",$", MR_Next},
 201            {"\"Bar\": %float,$", MR_Next},
 202            {"\"Bat\": %float,$", MR_Next},
 203            {"\"Baz\": %float,$", MR_Next},
 204            {"\"Foo\": %float,$", MR_Next},
 205            {"\"Frob\": %float,$", MR_Next},
 206            {"\"Lob\": %float$", MR_Next},
 207            {"}", MR_Next}});
 208
 209 ADD_CASES(TC_JSONOut,
 210           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
 211            {"\"family_index\": 0,$", MR_Next},
 212            {"\"per_family_instance_index\": 1,$", MR_Next},
 213            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 214             MR_Next},
 215            {"\"run_type\": \"iteration\",$", MR_Next},
 216            {"\"repetitions\": 2,$", MR_Next},
 217            {"\"repetition_index\": 0,$", MR_Next},
 218            {"\"threads\": 2,$", MR_Next},
 219            {"\"iterations\": %int,$", MR_Next},
 220            {"\"real_time\": %float,$", MR_Next},
 221            {"\"cpu_time\": %float,$", MR_Next},
 222            {"\"time_unit\": \"ns\",$", MR_Next},
 223            {"\"Bar\": %float,$", MR_Next},
 224            {"\"Bat\": %float,$", MR_Next},
 225            {"\"Baz\": %float,$", MR_Next},
 226            {"\"Foo\": %float,$", MR_Next},
 227            {"\"Frob\": %float,$", MR_Next},
 228            {"\"Lob\": %float$", MR_Next},
 229            {"}", MR_Next}});
 230 ADD_CASES(TC_JSONOut,
 231           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
 232            {"\"family_index\": 0,$", MR_Next},
 233            {"\"per_family_instance_index\": 1,$", MR_Next},
 234            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 235             MR_Next},
 236            {"\"run_type\": \"iteration\",$", MR_Next},
 237            {"\"repetitions\": 2,$", MR_Next},
 238            {"\"repetition_index\": 1,$", MR_Next},
 239            {"\"threads\": 2,$", MR_Next},
 240            {"\"iterations\": %int,$", MR_Next},
 241            {"\"real_time\": %float,$", MR_Next},
 242            {"\"cpu_time\": %float,$", MR_Next},
 243            {"\"time_unit\": \"ns\",$", MR_Next},
 244            {"\"Bar\": %float,$", MR_Next},
 245            {"\"Bat\": %float,$", MR_Next},
 246            {"\"Baz\": %float,$", MR_Next},
 247            {"\"Foo\": %float,$", MR_Next},
 248            {"\"Frob\": %float,$", MR_Next},
 249            {"\"Lob\": %float$", MR_Next},
 250            {"}", MR_Next}});
 251 ADD_CASES(TC_JSONOut,
 252           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"},
 253            {"\"family_index\": 0,$", MR_Next},
 254            {"\"per_family_instance_index\": 1,$", MR_Next},
 255            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 256             MR_Next},
 257            {"\"run_type\": \"aggregate\",$", MR_Next},
 258            {"\"repetitions\": 2,$", MR_Next},
 259            {"\"threads\": 2,$", MR_Next},
 260            {"\"aggregate_name\": \"median\",$", MR_Next},
 261            {"\"aggregate_unit\": \"time\",$", MR_Next},
 262            {"\"iterations\": %int,$", MR_Next},
 263            {"\"real_time\": %float,$", MR_Next},
 264            {"\"cpu_time\": %float,$", MR_Next},
 265            {"\"time_unit\": \"ns\",$", MR_Next},
 266            {"\"Bar\": %float,$", MR_Next},
 267            {"\"Bat\": %float,$", MR_Next},
 268            {"\"Baz\": %float,$", MR_Next},
 269            {"\"Foo\": %float,$", MR_Next},
 270            {"\"Frob\": %float,$", MR_Next},
 271            {"\"Lob\": %float$", MR_Next},
 272            {"}", MR_Next}});
 273 ADD_CASES(TC_JSONOut,
 274           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
 275            {"\"family_index\": 0,$", MR_Next},
 276            {"\"per_family_instance_index\": 1,$", MR_Next},
 277            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 278             MR_Next},
 279            {"\"run_type\": \"aggregate\",$", MR_Next},
 280            {"\"repetitions\": 2,$", MR_Next},
 281            {"\"threads\": 2,$", MR_Next},
 282            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 283            {"\"aggregate_unit\": \"time\",$", MR_Next},
 284            {"\"iterations\": %int,$", MR_Next},
 285            {"\"real_time\": %float,$", MR_Next},
 286            {"\"cpu_time\": %float,$", MR_Next},
 287            {"\"time_unit\": \"ns\",$", MR_Next},
 288            {"\"Bar\": %float,$", MR_Next},
 289            {"\"Bat\": %float,$", MR_Next},
 290            {"\"Baz\": %float,$", MR_Next},
 291            {"\"Foo\": %float,$", MR_Next},
 292            {"\"Frob\": %float,$", MR_Next},
 293            {"\"Lob\": %float$", MR_Next},
 294            {"}", MR_Next}});
 295 ADD_CASES(TC_JSONOut,
 296           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
 297            {"\"family_index\": 0,$", MR_Next},
 298            {"\"per_family_instance_index\": 1,$", MR_Next},
 299            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 300             MR_Next},
 301            {"\"run_type\": \"aggregate\",$", MR_Next},
 302            {"\"repetitions\": 2,$", MR_Next},
 303            {"\"threads\": 2,$", MR_Next},
 304            {"\"aggregate_name\": \"cv\",$", MR_Next},
 305            {"\"aggregate_unit\": \"percentage\",$", MR_Next},
 306            {"\"iterations\": %int,$", MR_Next},
 307            {"\"real_time\": %float,$", MR_Next},
 308            {"\"cpu_time\": %float,$", MR_Next},
 309            {"\"time_unit\": \"ns\",$", MR_Next},
 310            {"\"Bar\": %float,$", MR_Next},
 311            {"\"Bat\": %float,$", MR_Next},
 312            {"\"Baz\": %float,$", MR_Next},
 313            {"\"Foo\": %float,$", MR_Next},
 314            {"\"Frob\": %float,$", MR_Next},
 315            {"\"Lob\": %float$", MR_Next},
 316            {"}", MR_Next}});
 317 ADD_CASES(TC_CSVOut,
 318           {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
 319             "%float,%float,%float,%float,%float,%float$"}});
 320 ADD_CASES(TC_CSVOut,
 321           {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
 322             "%float,%float,%float,%float,%float,%float$"}});
 323 ADD_CASES(TC_CSVOut,
 324           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report,"
 325             "%float,%float,%float,%float,%float,%float$"}});
 326 ADD_CASES(TC_CSVOut,
 327           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
 328             "%float,%float,%float,%float,%float,%float$"}});
 329 ADD_CASES(TC_CSVOut,
 330           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
 331             "%float,%float,%float,%float,%float,%float$"}});
 332 ADD_CASES(TC_CSVOut,
 333           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report,"
 334             "%float,%float,%float,%float,%float,%float$"}});
 335 ADD_CASES(TC_CSVOut,
 336           {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
 337             "%float,%float,%float,%float,%float,%float$"}});
 338 ADD_CASES(TC_CSVOut,
 339           {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
 340             "%float,%float,%float,%float,%float,%float$"}});
 341 ADD_CASES(TC_CSVOut,
 342           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
 343             "%float,%float,%float,%float,%float,%float$"}});
 344 ADD_CASES(TC_CSVOut,
 345           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
 346             "%float,%float,%float,%float,%float,%float$"}});
 347 ADD_CASES(TC_CSVOut,
 348           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
 349             "%float,%float,%float,%float,%float,%float$"}});
 350 ADD_CASES(TC_CSVOut,
 351           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report,"
 352             "%float,%float,%float,%float,%float,%float$"}});
 353 // VS2013 does not allow this function to be passed as a lambda argument
 354 // to CHECK_BENCHMARK_RESULTS()
 355 void CheckTabular(Results const& e) {
 356   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
 357   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
 358   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
 359   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
 360   CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
 361   CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
 362 }
 363 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$",
 364                         &CheckTabular);
 365 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$",
 366                         &CheckTabular);
 367
 368 // ========================================================================= //
 369 // -------------------- Tabular+Rate Counters Output ----------------------- //
 370 // ========================================================================= //
 371
 372 void BM_CounterRates_Tabular(benchmark::State& state) {
 373   for (auto _ : state) {
 374     // This test requires a non-zero CPU time to avoid divide-by-zero
 375     benchmark::DoNotOptimize(state.iterations());
 376   }
 377   namespace bm = benchmark;
 378   state.counters.insert({
 379       {"Foo", {1, bm::Counter::kAvgThreadsRate}},
 380       {"Bar", {2, bm::Counter::kAvgThreadsRate}},
 381       {"Baz", {4, bm::Counter::kAvgThreadsRate}},
 382       {"Bat", {8, bm::Counter::kAvgThreadsRate}},
 383       {"Frob", {16, bm::Counter::kAvgThreadsRate}},
 384       {"Lob", {32, bm::Counter::kAvgThreadsRate}},
 385   });
 386 }
 387 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
 388 ADD_CASES(TC_JSONOut,
 389           {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
 390            {"\"family_index\": 1,$", MR_Next},
 391            {"\"per_family_instance_index\": 0,$", MR_Next},
 392            {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
 393             MR_Next},
 394            {"\"run_type\": \"iteration\",$", MR_Next},
 395            {"\"repetitions\": 1,$", MR_Next},
 396            {"\"repetition_index\": 0,$", MR_Next},
 397            {"\"threads\": 1,$", MR_Next},
 398            {"\"iterations\": %int,$", MR_Next},
 399            {"\"real_time\": %float,$", MR_Next},
 400            {"\"cpu_time\": %float,$", MR_Next},
 401            {"\"time_unit\": \"ns\",$", MR_Next},
 402            {"\"Bar\": %float,$", MR_Next},
 403            {"\"Bat\": %float,$", MR_Next},
 404            {"\"Baz\": %float,$", MR_Next},
 405            {"\"Foo\": %float,$", MR_Next},
 406            {"\"Frob\": %float,$", MR_Next},
 407            {"\"Lob\": %float$", MR_Next},
 408            {"}", MR_Next}});
 409 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
 410                        "%float,%float,%float,%float,%float,%float$"}});
 411 // VS2013 does not allow this function to be passed as a lambda argument
 412 // to CHECK_BENCHMARK_RESULTS()
 413 void CheckTabularRate(Results const& e) {
 414   double t = e.DurationCPUTime();
 415   CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
 416   CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
 417   CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
 418   CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
 419   CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
 420   CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
 421 }
 422 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
 423                         &CheckTabularRate);
 424
 425 // ========================================================================= //
 426 // ------------------------- Tabular Counters Output ----------------------- //
 427 // ========================================================================= //
 428
 429 // set only some of the counters
 430 void BM_CounterSet0_Tabular(benchmark::State& state) {
 431   for (auto _ : state) {
 432   }
 433   namespace bm = benchmark;
 434   state.counters.insert({
 435       {"Foo", {10, bm::Counter::kAvgThreads}},
 436       {"Bar", {20, bm::Counter::kAvgThreads}},
 437       {"Baz", {40, bm::Counter::kAvgThreads}},
 438   });
 439 }
 440 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
 441 ADD_CASES(TC_JSONOut,
 442           {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
 443            {"\"family_index\": 2,$", MR_Next},
 444            {"\"per_family_instance_index\": 0,$", MR_Next},
 445            {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
 446            {"\"run_type\": \"iteration\",$", MR_Next},
 447            {"\"repetitions\": 1,$", MR_Next},
 448            {"\"repetition_index\": 0,$", MR_Next},
 449            {"\"threads\": 1,$", MR_Next},
 450            {"\"iterations\": %int,$", MR_Next},
 451            {"\"real_time\": %float,$", MR_Next},
 452            {"\"cpu_time\": %float,$", MR_Next},
 453            {"\"time_unit\": \"ns\",$", MR_Next},
 454            {"\"Bar\": %float,$", MR_Next},
 455            {"\"Baz\": %float,$", MR_Next},
 456            {"\"Foo\": %float$", MR_Next},
 457            {"}", MR_Next}});
 458 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
 459                        "%float,,%float,%float,,"}});
 460 // VS2013 does not allow this function to be passed as a lambda argument
 461 // to CHECK_BENCHMARK_RESULTS()
 462 void CheckSet0(Results const& e) {
 463   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 464   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
 465   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 466 }
 467 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
 468
 469 // again.
 470 void BM_CounterSet1_Tabular(benchmark::State& state) {
 471   for (auto _ : state) {
 472   }
 473   namespace bm = benchmark;
 474   state.counters.insert({
 475       {"Foo", {15, bm::Counter::kAvgThreads}},
 476       {"Bar", {25, bm::Counter::kAvgThreads}},
 477       {"Baz", {45, bm::Counter::kAvgThreads}},
 478   });
 479 }
 480 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
 481 ADD_CASES(TC_JSONOut,
 482           {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
 483            {"\"family_index\": 3,$", MR_Next},
 484            {"\"per_family_instance_index\": 0,$", MR_Next},
 485            {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
 486            {"\"run_type\": \"iteration\",$", MR_Next},
 487            {"\"repetitions\": 1,$", MR_Next},
 488            {"\"repetition_index\": 0,$", MR_Next},
 489            {"\"threads\": 1,$", MR_Next},
 490            {"\"iterations\": %int,$", MR_Next},
 491            {"\"real_time\": %float,$", MR_Next},
 492            {"\"cpu_time\": %float,$", MR_Next},
 493            {"\"time_unit\": \"ns\",$", MR_Next},
 494            {"\"Bar\": %float,$", MR_Next},
 495            {"\"Baz\": %float,$", MR_Next},
 496            {"\"Foo\": %float$", MR_Next},
 497            {"}", MR_Next}});
 498 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
 499                        "%float,,%float,%float,,"}});
 500 // VS2013 does not allow this function to be passed as a lambda argument
 501 // to CHECK_BENCHMARK_RESULTS()
 502 void CheckSet1(Results const& e) {
 503   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
 504   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
 505   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
 506 }
 507 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
 508
 509 // ========================================================================= //
 510 // ------------------------- Tabular Counters Output ----------------------- //
 511 // ========================================================================= //
 512
 513 // set only some of the counters, different set now.
 514 void BM_CounterSet2_Tabular(benchmark::State& state) {
 515   for (auto _ : state) {
 516   }
 517   namespace bm = benchmark;
 518   state.counters.insert({
 519       {"Foo", {10, bm::Counter::kAvgThreads}},
 520       {"Bat", {30, bm::Counter::kAvgThreads}},
 521       {"Baz", {40, bm::Counter::kAvgThreads}},
 522   });
 523 }
 524 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
 525 ADD_CASES(TC_JSONOut,
 526           {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
 527            {"\"family_index\": 4,$", MR_Next},
 528            {"\"per_family_instance_index\": 0,$", MR_Next},
 529            {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
 530            {"\"run_type\": \"iteration\",$", MR_Next},
 531            {"\"repetitions\": 1,$", MR_Next},
 532            {"\"repetition_index\": 0,$", MR_Next},
 533            {"\"threads\": 1,$", MR_Next},
 534            {"\"iterations\": %int,$", MR_Next},
 535            {"\"real_time\": %float,$", MR_Next},
 536            {"\"cpu_time\": %float,$", MR_Next},
 537            {"\"time_unit\": \"ns\",$", MR_Next},
 538            {"\"Bat\": %float,$", MR_Next},
 539            {"\"Baz\": %float,$", MR_Next},
 540            {"\"Foo\": %float$", MR_Next},
 541            {"}", MR_Next}});
 542 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
 543                        ",%float,%float,%float,,"}});
 544 // VS2013 does not allow this function to be passed as a lambda argument
 545 // to CHECK_BENCHMARK_RESULTS()
 546 void CheckSet2(Results const& e) {
 547   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 548   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
 549   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 550 }
 551 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
 552
 553 // ========================================================================= //
 554 // --------------------------- TEST CASES END ------------------------------ //
 555 // ========================================================================= //
 556
 557 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }