third-party/benchmark/test/user_counters_tabular_test.cc

   1
   2 #undef NDEBUG
   3
   4 #include "benchmark/benchmark.h"
   5 #include "output_test.h"
   6
   7 // @todo: <jpmag> this checks the full output at once; the rule for
   8 // CounterSet1 was failing because it was not matching "^[-]+$".
   9 // @todo: <jpmag> check that the counters are vertically aligned.
  10 ADD_CASES(TC_ConsoleOut,
  11           {
  12               // keeping these lines long improves readability, so:
  13               // clang-format off
  14     {"^[-]+$", MR_Next},
  15     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
  16     {"^[-]+$", MR_Next},
  17       {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  18       {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  19       {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  20       {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  21             {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  22             {"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
  23       {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  24       {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  25       {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  26       {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  27             {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  28             {"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
  29     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  30     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  31     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  32     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  33     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
  34     {"^[-]+$", MR_Next},
  35     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
  36     {"^[-]+$", MR_Next},
  37     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  38     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  39     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  40     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  41     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  42     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  43     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  44     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  45     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  46     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  47     {"^[-]+$", MR_Next},
  48     {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
  49     {"^[-]+$", MR_Next},
  50     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  51     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  52     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  53     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
  54     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
  55               // clang-format on
  56           });
  57 ADD_CASES(TC_CSVOut, {{"%csv_header,"
  58                        "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
  59
  60 // ========================================================================= //
  61 // ------------------------- Tabular Counters Output ----------------------- //
  62 // ========================================================================= //
  63
  64 void BM_Counters_Tabular(benchmark::State& state) {
  65   for (auto _ : state) {
  66     // This test requires a non-zero CPU time to avoid divide-by-zero
  67     auto iterations = double(state.iterations()) * double(state.iterations());
  68     benchmark::DoNotOptimize(iterations);
  69   }
  70   namespace bm = benchmark;
  71   state.counters.insert({
  72       {"Foo", {1, bm::Counter::kAvgThreads}},
  73       {"Bar", {2, bm::Counter::kAvgThreads}},
  74       {"Baz", {4, bm::Counter::kAvgThreads}},
  75       {"Bat", {8, bm::Counter::kAvgThreads}},
  76       {"Frob", {16, bm::Counter::kAvgThreads}},
  77       {"Lob", {32, bm::Counter::kAvgThreads}},
  78   });
  79 }
  80 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);
  81 ADD_CASES(TC_JSONOut,
  82           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
  83            {"\"family_index\": 0,$", MR_Next},
  84            {"\"per_family_instance_index\": 0,$", MR_Next},
  85            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
  86             MR_Next},
  87            {"\"run_type\": \"iteration\",$", MR_Next},
  88            {"\"repetitions\": 2,$", MR_Next},
  89            {"\"repetition_index\": 0,$", MR_Next},
  90            {"\"threads\": 1,$", MR_Next},
  91            {"\"iterations\": %int,$", MR_Next},
  92            {"\"real_time\": %float,$", MR_Next},
  93            {"\"cpu_time\": %float,$", MR_Next},
  94            {"\"time_unit\": \"ns\",$", MR_Next},
  95            {"\"Bar\": %float,$", MR_Next},
  96            {"\"Bat\": %float,$", MR_Next},
  97            {"\"Baz\": %float,$", MR_Next},
  98            {"\"Foo\": %float,$", MR_Next},
  99            {"\"Frob\": %float,$", MR_Next},
 100            {"\"Lob\": %float$", MR_Next},
 101            {"}", MR_Next}});
 102 ADD_CASES(TC_JSONOut,
 103           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
 104            {"\"family_index\": 0,$", MR_Next},
 105            {"\"per_family_instance_index\": 0,$", MR_Next},
 106            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 107             MR_Next},
 108            {"\"run_type\": \"iteration\",$", MR_Next},
 109            {"\"repetitions\": 2,$", MR_Next},
 110            {"\"repetition_index\": 1,$", MR_Next},
 111            {"\"threads\": 1,$", MR_Next},
 112            {"\"iterations\": %int,$", MR_Next},
 113            {"\"real_time\": %float,$", MR_Next},
 114            {"\"cpu_time\": %float,$", MR_Next},
 115            {"\"time_unit\": \"ns\",$", MR_Next},
 116            {"\"Bar\": %float,$", MR_Next},
 117            {"\"Bat\": %float,$", MR_Next},
 118            {"\"Baz\": %float,$", MR_Next},
 119            {"\"Foo\": %float,$", MR_Next},
 120            {"\"Frob\": %float,$", MR_Next},
 121            {"\"Lob\": %float$", MR_Next},
 122            {"}", MR_Next}});
 123 ADD_CASES(TC_JSONOut,
 124           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"},
 125            {"\"family_index\": 0,$", MR_Next},
 126            {"\"per_family_instance_index\": 0,$", MR_Next},
 127            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 128             MR_Next},
 129            {"\"run_type\": \"aggregate\",$", MR_Next},
 130            {"\"repetitions\": 2,$", MR_Next},
 131            {"\"threads\": 1,$", MR_Next},
 132            {"\"aggregate_name\": \"mean\",$", MR_Next},
 133            {"\"aggregate_unit\": \"time\",$", MR_Next},
 134            {"\"iterations\": %int,$", MR_Next},
 135            {"\"real_time\": %float,$", MR_Next},
 136            {"\"cpu_time\": %float,$", MR_Next},
 137            {"\"time_unit\": \"ns\",$", MR_Next},
 138            {"\"Bar\": %float,$", MR_Next},
 139            {"\"Bat\": %float,$", MR_Next},
 140            {"\"Baz\": %float,$", MR_Next},
 141            {"\"Foo\": %float,$", MR_Next},
 142            {"\"Frob\": %float,$", MR_Next},
 143            {"\"Lob\": %float$", MR_Next},
 144            {"}", MR_Next}});
 145 ADD_CASES(TC_JSONOut,
 146           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
 147            {"\"family_index\": 0,$", MR_Next},
 148            {"\"per_family_instance_index\": 0,$", MR_Next},
 149            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 150             MR_Next},
 151            {"\"run_type\": \"aggregate\",$", MR_Next},
 152            {"\"repetitions\": 2,$", MR_Next},
 153            {"\"threads\": 1,$", MR_Next},
 154            {"\"aggregate_name\": \"median\",$", MR_Next},
 155            {"\"aggregate_unit\": \"time\",$", MR_Next},
 156            {"\"iterations\": %int,$", MR_Next},
 157            {"\"real_time\": %float,$", MR_Next},
 158            {"\"cpu_time\": %float,$", MR_Next},
 159            {"\"time_unit\": \"ns\",$", MR_Next},
 160            {"\"Bar\": %float,$", MR_Next},
 161            {"\"Bat\": %float,$", MR_Next},
 162            {"\"Baz\": %float,$", MR_Next},
 163            {"\"Foo\": %float,$", MR_Next},
 164            {"\"Frob\": %float,$", MR_Next},
 165            {"\"Lob\": %float$", MR_Next},
 166            {"}", MR_Next}});
 167 ADD_CASES(TC_JSONOut,
 168           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
 169            {"\"family_index\": 0,$", MR_Next},
 170            {"\"per_family_instance_index\": 0,$", MR_Next},
 171            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 172             MR_Next},
 173            {"\"run_type\": \"aggregate\",$", MR_Next},
 174            {"\"repetitions\": 2,$", MR_Next},
 175            {"\"threads\": 1,$", MR_Next},
 176            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 177            {"\"aggregate_unit\": \"time\",$", MR_Next},
 178            {"\"iterations\": %int,$", MR_Next},
 179            {"\"real_time\": %float,$", MR_Next},
 180            {"\"cpu_time\": %float,$", MR_Next},
 181            {"\"time_unit\": \"ns\",$", MR_Next},
 182            {"\"Bar\": %float,$", MR_Next},
 183            {"\"Bat\": %float,$", MR_Next},
 184            {"\"Baz\": %float,$", MR_Next},
 185            {"\"Foo\": %float,$", MR_Next},
 186            {"\"Frob\": %float,$", MR_Next},
 187            {"\"Lob\": %float$", MR_Next},
 188            {"}", MR_Next}});
 189 ADD_CASES(TC_JSONOut,
 190           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
 191            {"\"family_index\": 0,$", MR_Next},
 192            {"\"per_family_instance_index\": 0,$", MR_Next},
 193            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
 194             MR_Next},
 195            {"\"run_type\": \"aggregate\",$", MR_Next},
 196            {"\"repetitions\": 2,$", MR_Next},
 197            {"\"threads\": 1,$", MR_Next},
 198            {"\"aggregate_name\": \"cv\",$", MR_Next},
 199            {"\"aggregate_unit\": \"percentage\",$", MR_Next},
 200            {"\"iterations\": %int,$", MR_Next},
 201            {"\"real_time\": %float,$", MR_Next},
 202            {"\"cpu_time\": %float,$", MR_Next},
 203            {"\"time_unit\": \"ns\",$", MR_Next},
 204            {"\"Bar\": %float,$", MR_Next},
 205            {"\"Bat\": %float,$", MR_Next},
 206            {"\"Baz\": %float,$", MR_Next},
 207            {"\"Foo\": %float,$", MR_Next},
 208            {"\"Frob\": %float,$", MR_Next},
 209            {"\"Lob\": %float$", MR_Next},
 210            {"}", MR_Next}});
 211
 212 ADD_CASES(TC_JSONOut,
 213           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
 214            {"\"family_index\": 0,$", MR_Next},
 215            {"\"per_family_instance_index\": 1,$", MR_Next},
 216            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 217             MR_Next},
 218            {"\"run_type\": \"iteration\",$", MR_Next},
 219            {"\"repetitions\": 2,$", MR_Next},
 220            {"\"repetition_index\": 0,$", MR_Next},
 221            {"\"threads\": 2,$", MR_Next},
 222            {"\"iterations\": %int,$", MR_Next},
 223            {"\"real_time\": %float,$", MR_Next},
 224            {"\"cpu_time\": %float,$", MR_Next},
 225            {"\"time_unit\": \"ns\",$", MR_Next},
 226            {"\"Bar\": %float,$", MR_Next},
 227            {"\"Bat\": %float,$", MR_Next},
 228            {"\"Baz\": %float,$", MR_Next},
 229            {"\"Foo\": %float,$", MR_Next},
 230            {"\"Frob\": %float,$", MR_Next},
 231            {"\"Lob\": %float$", MR_Next},
 232            {"}", MR_Next}});
 233 ADD_CASES(TC_JSONOut,
 234           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
 235            {"\"family_index\": 0,$", MR_Next},
 236            {"\"per_family_instance_index\": 1,$", MR_Next},
 237            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 238             MR_Next},
 239            {"\"run_type\": \"iteration\",$", MR_Next},
 240            {"\"repetitions\": 2,$", MR_Next},
 241            {"\"repetition_index\": 1,$", MR_Next},
 242            {"\"threads\": 2,$", MR_Next},
 243            {"\"iterations\": %int,$", MR_Next},
 244            {"\"real_time\": %float,$", MR_Next},
 245            {"\"cpu_time\": %float,$", MR_Next},
 246            {"\"time_unit\": \"ns\",$", MR_Next},
 247            {"\"Bar\": %float,$", MR_Next},
 248            {"\"Bat\": %float,$", MR_Next},
 249            {"\"Baz\": %float,$", MR_Next},
 250            {"\"Foo\": %float,$", MR_Next},
 251            {"\"Frob\": %float,$", MR_Next},
 252            {"\"Lob\": %float$", MR_Next},
 253            {"}", MR_Next}});
 254 ADD_CASES(TC_JSONOut,
 255           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"},
 256            {"\"family_index\": 0,$", MR_Next},
 257            {"\"per_family_instance_index\": 1,$", MR_Next},
 258            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 259             MR_Next},
 260            {"\"run_type\": \"aggregate\",$", MR_Next},
 261            {"\"repetitions\": 2,$", MR_Next},
 262            {"\"threads\": 2,$", MR_Next},
 263            {"\"aggregate_name\": \"median\",$", MR_Next},
 264            {"\"aggregate_unit\": \"time\",$", MR_Next},
 265            {"\"iterations\": %int,$", MR_Next},
 266            {"\"real_time\": %float,$", MR_Next},
 267            {"\"cpu_time\": %float,$", MR_Next},
 268            {"\"time_unit\": \"ns\",$", MR_Next},
 269            {"\"Bar\": %float,$", MR_Next},
 270            {"\"Bat\": %float,$", MR_Next},
 271            {"\"Baz\": %float,$", MR_Next},
 272            {"\"Foo\": %float,$", MR_Next},
 273            {"\"Frob\": %float,$", MR_Next},
 274            {"\"Lob\": %float$", MR_Next},
 275            {"}", MR_Next}});
 276 ADD_CASES(TC_JSONOut,
 277           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
 278            {"\"family_index\": 0,$", MR_Next},
 279            {"\"per_family_instance_index\": 1,$", MR_Next},
 280            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 281             MR_Next},
 282            {"\"run_type\": \"aggregate\",$", MR_Next},
 283            {"\"repetitions\": 2,$", MR_Next},
 284            {"\"threads\": 2,$", MR_Next},
 285            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 286            {"\"aggregate_unit\": \"time\",$", MR_Next},
 287            {"\"iterations\": %int,$", MR_Next},
 288            {"\"real_time\": %float,$", MR_Next},
 289            {"\"cpu_time\": %float,$", MR_Next},
 290            {"\"time_unit\": \"ns\",$", MR_Next},
 291            {"\"Bar\": %float,$", MR_Next},
 292            {"\"Bat\": %float,$", MR_Next},
 293            {"\"Baz\": %float,$", MR_Next},
 294            {"\"Foo\": %float,$", MR_Next},
 295            {"\"Frob\": %float,$", MR_Next},
 296            {"\"Lob\": %float$", MR_Next},
 297            {"}", MR_Next}});
 298 ADD_CASES(TC_JSONOut,
 299           {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
 300            {"\"family_index\": 0,$", MR_Next},
 301            {"\"per_family_instance_index\": 1,$", MR_Next},
 302            {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
 303             MR_Next},
 304            {"\"run_type\": \"aggregate\",$", MR_Next},
 305            {"\"repetitions\": 2,$", MR_Next},
 306            {"\"threads\": 2,$", MR_Next},
 307            {"\"aggregate_name\": \"cv\",$", MR_Next},
 308            {"\"aggregate_unit\": \"percentage\",$", MR_Next},
 309            {"\"iterations\": %int,$", MR_Next},
 310            {"\"real_time\": %float,$", MR_Next},
 311            {"\"cpu_time\": %float,$", MR_Next},
 312            {"\"time_unit\": \"ns\",$", MR_Next},
 313            {"\"Bar\": %float,$", MR_Next},
 314            {"\"Bat\": %float,$", MR_Next},
 315            {"\"Baz\": %float,$", MR_Next},
 316            {"\"Foo\": %float,$", MR_Next},
 317            {"\"Frob\": %float,$", MR_Next},
 318            {"\"Lob\": %float$", MR_Next},
 319            {"}", MR_Next}});
 320 ADD_CASES(TC_CSVOut,
 321           {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
 322             "%float,%float,%float,%float,%float,%float$"}});
 323 ADD_CASES(TC_CSVOut,
 324           {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
 325             "%float,%float,%float,%float,%float,%float$"}});
 326 ADD_CASES(TC_CSVOut,
 327           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report,"
 328             "%float,%float,%float,%float,%float,%float$"}});
 329 ADD_CASES(TC_CSVOut,
 330           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
 331             "%float,%float,%float,%float,%float,%float$"}});
 332 ADD_CASES(TC_CSVOut,
 333           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
 334             "%float,%float,%float,%float,%float,%float$"}});
 335 ADD_CASES(TC_CSVOut,
 336           {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report,"
 337             "%float,%float,%float,%float,%float,%float$"}});
 338 ADD_CASES(TC_CSVOut,
 339           {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
 340             "%float,%float,%float,%float,%float,%float$"}});
 341 ADD_CASES(TC_CSVOut,
 342           {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
 343             "%float,%float,%float,%float,%float,%float$"}});
 344 ADD_CASES(TC_CSVOut,
 345           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
 346             "%float,%float,%float,%float,%float,%float$"}});
 347 ADD_CASES(TC_CSVOut,
 348           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
 349             "%float,%float,%float,%float,%float,%float$"}});
 350 ADD_CASES(TC_CSVOut,
 351           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
 352             "%float,%float,%float,%float,%float,%float$"}});
 353 ADD_CASES(TC_CSVOut,
 354           {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report,"
 355             "%float,%float,%float,%float,%float,%float$"}});
 356 // VS2013 does not allow this function to be passed as a lambda argument
 357 // to CHECK_BENCHMARK_RESULTS()
 358 void CheckTabular(Results const& e) {
 359   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
 360   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
 361   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
 362   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
 363   CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
 364   CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
 365 }
 366 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$",
 367                         &CheckTabular);
 368 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$",
 369                         &CheckTabular);
 370
 371 // ========================================================================= //
 372 // -------------------- Tabular+Rate Counters Output ----------------------- //
 373 // ========================================================================= //
 374
 375 void BM_CounterRates_Tabular(benchmark::State& state) {
 376   for (auto _ : state) {
 377     // This test requires a non-zero CPU time to avoid divide-by-zero
 378     auto iterations = double(state.iterations()) * double(state.iterations());
 379     benchmark::DoNotOptimize(iterations);
 380   }
 381   namespace bm = benchmark;
 382   state.counters.insert({
 383       {"Foo", {1, bm::Counter::kAvgThreadsRate}},
 384       {"Bar", {2, bm::Counter::kAvgThreadsRate}},
 385       {"Baz", {4, bm::Counter::kAvgThreadsRate}},
 386       {"Bat", {8, bm::Counter::kAvgThreadsRate}},
 387       {"Frob", {16, bm::Counter::kAvgThreadsRate}},
 388       {"Lob", {32, bm::Counter::kAvgThreadsRate}},
 389   });
 390 }
 391 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
 392 ADD_CASES(TC_JSONOut,
 393           {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
 394            {"\"family_index\": 1,$", MR_Next},
 395            {"\"per_family_instance_index\": 0,$", MR_Next},
 396            {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
 397             MR_Next},
 398            {"\"run_type\": \"iteration\",$", MR_Next},
 399            {"\"repetitions\": 1,$", MR_Next},
 400            {"\"repetition_index\": 0,$", MR_Next},
 401            {"\"threads\": 1,$", MR_Next},
 402            {"\"iterations\": %int,$", MR_Next},
 403            {"\"real_time\": %float,$", MR_Next},
 404            {"\"cpu_time\": %float,$", MR_Next},
 405            {"\"time_unit\": \"ns\",$", MR_Next},
 406            {"\"Bar\": %float,$", MR_Next},
 407            {"\"Bat\": %float,$", MR_Next},
 408            {"\"Baz\": %float,$", MR_Next},
 409            {"\"Foo\": %float,$", MR_Next},
 410            {"\"Frob\": %float,$", MR_Next},
 411            {"\"Lob\": %float$", MR_Next},
 412            {"}", MR_Next}});
 413 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
 414                        "%float,%float,%float,%float,%float,%float$"}});
 415 // VS2013 does not allow this function to be passed as a lambda argument
 416 // to CHECK_BENCHMARK_RESULTS()
 417 void CheckTabularRate(Results const& e) {
 418   double t = e.DurationCPUTime();
 419   CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
 420   CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
 421   CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
 422   CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
 423   CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
 424   CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
 425 }
 426 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
 427                         &CheckTabularRate);
 428
 429 // ========================================================================= //
 430 // ------------------------- Tabular Counters Output ----------------------- //
 431 // ========================================================================= //
 432
 433 // set only some of the counters
 434 void BM_CounterSet0_Tabular(benchmark::State& state) {
 435   for (auto _ : state) {
 436   }
 437   namespace bm = benchmark;
 438   state.counters.insert({
 439       {"Foo", {10, bm::Counter::kAvgThreads}},
 440       {"Bar", {20, bm::Counter::kAvgThreads}},
 441       {"Baz", {40, bm::Counter::kAvgThreads}},
 442   });
 443 }
 444 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
 445 ADD_CASES(TC_JSONOut,
 446           {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
 447            {"\"family_index\": 2,$", MR_Next},
 448            {"\"per_family_instance_index\": 0,$", MR_Next},
 449            {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
 450            {"\"run_type\": \"iteration\",$", MR_Next},
 451            {"\"repetitions\": 1,$", MR_Next},
 452            {"\"repetition_index\": 0,$", MR_Next},
 453            {"\"threads\": 1,$", MR_Next},
 454            {"\"iterations\": %int,$", MR_Next},
 455            {"\"real_time\": %float,$", MR_Next},
 456            {"\"cpu_time\": %float,$", MR_Next},
 457            {"\"time_unit\": \"ns\",$", MR_Next},
 458            {"\"Bar\": %float,$", MR_Next},
 459            {"\"Baz\": %float,$", MR_Next},
 460            {"\"Foo\": %float$", MR_Next},
 461            {"}", MR_Next}});
 462 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
 463                        "%float,,%float,%float,,"}});
 464 // VS2013 does not allow this function to be passed as a lambda argument
 465 // to CHECK_BENCHMARK_RESULTS()
 466 void CheckSet0(Results const& e) {
 467   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 468   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
 469   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 470 }
 471 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
 472
 473 // again.
 474 void BM_CounterSet1_Tabular(benchmark::State& state) {
 475   for (auto _ : state) {
 476   }
 477   namespace bm = benchmark;
 478   state.counters.insert({
 479       {"Foo", {15, bm::Counter::kAvgThreads}},
 480       {"Bar", {25, bm::Counter::kAvgThreads}},
 481       {"Baz", {45, bm::Counter::kAvgThreads}},
 482   });
 483 }
 484 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
 485 ADD_CASES(TC_JSONOut,
 486           {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
 487            {"\"family_index\": 3,$", MR_Next},
 488            {"\"per_family_instance_index\": 0,$", MR_Next},
 489            {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
 490            {"\"run_type\": \"iteration\",$", MR_Next},
 491            {"\"repetitions\": 1,$", MR_Next},
 492            {"\"repetition_index\": 0,$", MR_Next},
 493            {"\"threads\": 1,$", MR_Next},
 494            {"\"iterations\": %int,$", MR_Next},
 495            {"\"real_time\": %float,$", MR_Next},
 496            {"\"cpu_time\": %float,$", MR_Next},
 497            {"\"time_unit\": \"ns\",$", MR_Next},
 498            {"\"Bar\": %float,$", MR_Next},
 499            {"\"Baz\": %float,$", MR_Next},
 500            {"\"Foo\": %float$", MR_Next},
 501            {"}", MR_Next}});
 502 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
 503                        "%float,,%float,%float,,"}});
 504 // VS2013 does not allow this function to be passed as a lambda argument
 505 // to CHECK_BENCHMARK_RESULTS()
 506 void CheckSet1(Results const& e) {
 507   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
 508   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
 509   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
 510 }
 511 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
 512
 513 // ========================================================================= //
 514 // ------------------------- Tabular Counters Output ----------------------- //
 515 // ========================================================================= //
 516
 517 // set only some of the counters, different set now.
 518 void BM_CounterSet2_Tabular(benchmark::State& state) {
 519   for (auto _ : state) {
 520   }
 521   namespace bm = benchmark;
 522   state.counters.insert({
 523       {"Foo", {10, bm::Counter::kAvgThreads}},
 524       {"Bat", {30, bm::Counter::kAvgThreads}},
 525       {"Baz", {40, bm::Counter::kAvgThreads}},
 526   });
 527 }
 528 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
 529 ADD_CASES(TC_JSONOut,
 530           {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
 531            {"\"family_index\": 4,$", MR_Next},
 532            {"\"per_family_instance_index\": 0,$", MR_Next},
 533            {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
 534            {"\"run_type\": \"iteration\",$", MR_Next},
 535            {"\"repetitions\": 1,$", MR_Next},
 536            {"\"repetition_index\": 0,$", MR_Next},
 537            {"\"threads\": 1,$", MR_Next},
 538            {"\"iterations\": %int,$", MR_Next},
 539            {"\"real_time\": %float,$", MR_Next},
 540            {"\"cpu_time\": %float,$", MR_Next},
 541            {"\"time_unit\": \"ns\",$", MR_Next},
 542            {"\"Bat\": %float,$", MR_Next},
 543            {"\"Baz\": %float,$", MR_Next},
 544            {"\"Foo\": %float$", MR_Next},
 545            {"}", MR_Next}});
 546 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
 547                        ",%float,%float,%float,,"}});
 548 // VS2013 does not allow this function to be passed as a lambda argument
 549 // to CHECK_BENCHMARK_RESULTS()
 550 void CheckSet2(Results const& e) {
 551   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
 552   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
 553   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
 554 }
 555 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
 556
 557 // ========================================================================= //
 558 // --------------------------- TEST CASES END ------------------------------ //
 559 // ========================================================================= //
 560
 561 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }