utils/benchmark/test/reporter_output_test.cc

   1
   2 #undef NDEBUG
   3 #include <utility>
   4
   5 #include "benchmark/benchmark.h"
   6 #include "output_test.h"
   7
   8 // ========================================================================= //
   9 // ---------------------- Testing Prologue Output -------------------------- //
  10 // ========================================================================= //
  11
  12 ADD_CASES(TC_ConsoleOut,
  13           {{"^[-]+$", MR_Next},
  14            {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
  15            {"^[-]+$", MR_Next}});
  16 static int AddContextCases() {
  17   AddCases(TC_ConsoleErr,
  18            {
  19                {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
  20                {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
  21                {"Run on \\(%int X %float MHz CPU s\\)", MR_Next},
  22            });
  23   AddCases(TC_JSONOut, {{"^\\{", MR_Default},
  24                         {"\"context\":", MR_Next},
  25                         {"\"date\": \"", MR_Next},
  26                         {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next},
  27                         {"\"num_cpus\": %int,$", MR_Next},
  28                         {"\"mhz_per_cpu\": %float,$", MR_Next},
  29                         {"\"cpu_scaling_enabled\": ", MR_Next},
  30                         {"\"caches\": \\[$", MR_Next}});
  31   auto const& Caches = benchmark::CPUInfo::Get().caches;
  32   if (!Caches.empty()) {
  33     AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
  34   }
  35   for (size_t I = 0; I < Caches.size(); ++I) {
  36     std::string num_caches_str =
  37         Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
  38     AddCases(
  39         TC_ConsoleErr,
  40         {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}});
  41     AddCases(TC_JSONOut, {{"\\{$", MR_Next},
  42                           {"\"type\": \"", MR_Next},
  43                           {"\"level\": %int,$", MR_Next},
  44                           {"\"size\": %int,$", MR_Next},
  45                           {"\"num_sharing\": %int$", MR_Next},
  46                           {"}[,]{0,1}$", MR_Next}});
  47   }
  48
  49   AddCases(TC_JSONOut, {{"],$"}});
  50   return 0;
  51 }
  52 int dummy_register = AddContextCases();
  53 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
  54
  55 // ========================================================================= //
  56 // ------------------------ Testing Basic Output --------------------------- //
  57 // ========================================================================= //
  58
  59 void BM_basic(benchmark::State& state) {
  60   for (auto _ : state) {
  61   }
  62 }
  63 BENCHMARK(BM_basic);
  64
  65 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
  66 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
  67                        {"\"iterations\": %int,$", MR_Next},
  68                        {"\"real_time\": %float,$", MR_Next},
  69                        {"\"cpu_time\": %float,$", MR_Next},
  70                        {"\"time_unit\": \"ns\"$", MR_Next},
  71                        {"}", MR_Next}});
  72 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
  73
  74 // ========================================================================= //
  75 // ------------------------ Testing Bytes per Second Output ---------------- //
  76 // ========================================================================= //
  77
  78 void BM_bytes_per_second(benchmark::State& state) {
  79   for (auto _ : state) {
  80   }
  81   state.SetBytesProcessed(1);
  82 }
  83 BENCHMARK(BM_bytes_per_second);
  84
  85 ADD_CASES(TC_ConsoleOut,
  86           {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
  87 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
  88                        {"\"iterations\": %int,$", MR_Next},
  89                        {"\"real_time\": %float,$", MR_Next},
  90                        {"\"cpu_time\": %float,$", MR_Next},
  91                        {"\"time_unit\": \"ns\",$", MR_Next},
  92                        {"\"bytes_per_second\": %float$", MR_Next},
  93                        {"}", MR_Next}});
  94 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
  95
  96 // ========================================================================= //
  97 // ------------------------ Testing Items per Second Output ---------------- //
  98 // ========================================================================= //
  99
 100 void BM_items_per_second(benchmark::State& state) {
 101   for (auto _ : state) {
 102   }
 103   state.SetItemsProcessed(1);
 104 }
 105 BENCHMARK(BM_items_per_second);
 106
 107 ADD_CASES(TC_ConsoleOut,
 108           {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
 109 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
 110                        {"\"iterations\": %int,$", MR_Next},
 111                        {"\"real_time\": %float,$", MR_Next},
 112                        {"\"cpu_time\": %float,$", MR_Next},
 113                        {"\"time_unit\": \"ns\",$", MR_Next},
 114                        {"\"items_per_second\": %float$", MR_Next},
 115                        {"}", MR_Next}});
 116 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
 117
 118 // ========================================================================= //
 119 // ------------------------ Testing Label Output --------------------------- //
 120 // ========================================================================= //
 121
 122 void BM_label(benchmark::State& state) {
 123   for (auto _ : state) {
 124   }
 125   state.SetLabel("some label");
 126 }
 127 BENCHMARK(BM_label);
 128
 129 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
 130 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
 131                        {"\"iterations\": %int,$", MR_Next},
 132                        {"\"real_time\": %float,$", MR_Next},
 133                        {"\"cpu_time\": %float,$", MR_Next},
 134                        {"\"time_unit\": \"ns\",$", MR_Next},
 135                        {"\"label\": \"some label\"$", MR_Next},
 136                        {"}", MR_Next}});
 137 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
 138                        "label\"%csv_label_report_end$"}});
 139
 140 // ========================================================================= //
 141 // ------------------------ Testing Error Output --------------------------- //
 142 // ========================================================================= //
 143
 144 void BM_error(benchmark::State& state) {
 145   state.SkipWithError("message");
 146   for (auto _ : state) {
 147   }
 148 }
 149 BENCHMARK(BM_error);
 150 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
 151 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
 152                        {"\"error_occurred\": true,$", MR_Next},
 153                        {"\"error_message\": \"message\",$", MR_Next}});
 154
 155 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
 156
 157 // ========================================================================= //
 158 // ------------------------ Testing No Arg Name Output -----------------------
 159 // //
 160 // ========================================================================= //
 161
 162 void BM_no_arg_name(benchmark::State& state) {
 163   for (auto _ : state) {
 164   }
 165 }
 166 BENCHMARK(BM_no_arg_name)->Arg(3);
 167 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
 168 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
 169 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
 170
 171 // ========================================================================= //
 172 // ------------------------ Testing Arg Name Output ----------------------- //
 173 // ========================================================================= //
 174
 175 void BM_arg_name(benchmark::State& state) {
 176   for (auto _ : state) {
 177   }
 178 }
 179 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
 180 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
 181 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
 182 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
 183
 184 // ========================================================================= //
 185 // ------------------------ Testing Arg Names Output ----------------------- //
 186 // ========================================================================= //
 187
 188 void BM_arg_names(benchmark::State& state) {
 189   for (auto _ : state) {
 190   }
 191 }
 192 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
 193 ADD_CASES(TC_ConsoleOut,
 194           {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
 195 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
 196 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
 197
 198 // ========================================================================= //
 199 // ----------------------- Testing Complexity Output ----------------------- //
 200 // ========================================================================= //
 201
 202 void BM_Complexity_O1(benchmark::State& state) {
 203   for (auto _ : state) {
 204   }
 205   state.SetComplexityN(state.range(0));
 206 }
 207 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
 208 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
 209                    {"%RMS", "[ ]*[0-9]+ %"}});
 210 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
 211                           {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
 212
 213 // ========================================================================= //
 214 // ----------------------- Testing Aggregate Output ------------------------ //
 215 // ========================================================================= //
 216
 217 // Test that non-aggregate data is printed by default
 218 void BM_Repeat(benchmark::State& state) {
 219   for (auto _ : state) {
 220   }
 221 }
 222 // need two repetitions min to be able to output any aggregate output
 223 BENCHMARK(BM_Repeat)->Repetitions(2);
 224 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
 225                           {"^BM_Repeat/repeats:2 %console_report$"},
 226                           {"^BM_Repeat/repeats:2_mean %console_report$"},
 227                           {"^BM_Repeat/repeats:2_median %console_report$"},
 228                           {"^BM_Repeat/repeats:2_stddev %console_report$"}});
 229 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
 230                        {"\"name\": \"BM_Repeat/repeats:2\",$"},
 231                        {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
 232                        {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
 233                        {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}});
 234 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
 235                       {"^\"BM_Repeat/repeats:2\",%csv_report$"},
 236                       {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
 237                       {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
 238                       {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
 239 // but for two repetitions, mean and median is the same, so let's repeat..
 240 BENCHMARK(BM_Repeat)->Repetitions(3);
 241 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
 242                           {"^BM_Repeat/repeats:3 %console_report$"},
 243                           {"^BM_Repeat/repeats:3 %console_report$"},
 244                           {"^BM_Repeat/repeats:3_mean %console_report$"},
 245                           {"^BM_Repeat/repeats:3_median %console_report$"},
 246                           {"^BM_Repeat/repeats:3_stddev %console_report$"}});
 247 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
 248                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 249                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 250                        {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
 251                        {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
 252                        {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
 253 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
 254                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 255                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 256                       {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
 257                       {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
 258                       {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
 259 // median differs between even/odd number of repetitions, so just to be sure
 260 BENCHMARK(BM_Repeat)->Repetitions(4);
 261 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
 262                           {"^BM_Repeat/repeats:4 %console_report$"},
 263                           {"^BM_Repeat/repeats:4 %console_report$"},
 264                           {"^BM_Repeat/repeats:4 %console_report$"},
 265                           {"^BM_Repeat/repeats:4_mean %console_report$"},
 266                           {"^BM_Repeat/repeats:4_median %console_report$"},
 267                           {"^BM_Repeat/repeats:4_stddev %console_report$"}});
 268 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
 269                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 270                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 271                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 272                        {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
 273                        {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
 274                        {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}});
 275 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
 276                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 277                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 278                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 279                       {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
 280                       {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
 281                       {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
 282
 283 // Test that a non-repeated test still prints non-aggregate results even when
 284 // only-aggregate reports have been requested
 285 void BM_RepeatOnce(benchmark::State& state) {
 286   for (auto _ : state) {
 287   }
 288 }
 289 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
 290 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
 291 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
 292 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
 293
 294 // Test that non-aggregate data is not reported
 295 void BM_SummaryRepeat(benchmark::State& state) {
 296   for (auto _ : state) {
 297   }
 298 }
 299 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
 300 ADD_CASES(TC_ConsoleOut,
 301           {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 302            {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
 303            {"^BM_SummaryRepeat/repeats:3_median %console_report$"},
 304            {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
 305 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 306                        {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
 307                        {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
 308                        {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
 309 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 310                       {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
 311                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
 312                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
 313
 314 void BM_RepeatTimeUnit(benchmark::State& state) {
 315   for (auto _ : state) {
 316   }
 317 }
 318 BENCHMARK(BM_RepeatTimeUnit)
 319     ->Repetitions(3)
 320     ->ReportAggregatesOnly()
 321     ->Unit(benchmark::kMicrosecond);
 322 ADD_CASES(TC_ConsoleOut,
 323           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 324            {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
 325            {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
 326            {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
 327 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 328                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
 329                        {"\"time_unit\": \"us\",?$"},
 330                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
 331                        {"\"time_unit\": \"us\",?$"},
 332                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
 333                        {"\"time_unit\": \"us\",?$"}});
 334 ADD_CASES(TC_CSVOut,
 335           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 336            {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
 337            {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
 338            {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
 339
 340 // ========================================================================= //
 341 // -------------------- Testing user-provided statistics ------------------- //
 342 // ========================================================================= //
 343
 344 const auto UserStatistics = [](const std::vector<double>& v) {
 345   return v.back();
 346 };
 347 void BM_UserStats(benchmark::State& state) {
 348   for (auto _ : state) {
 349   }
 350 }
 351 BENCHMARK(BM_UserStats)
 352     ->Repetitions(3)
 353     ->ComputeStatistics("", UserStatistics);
 354 // check that user-provided stats is calculated, and is after the default-ones
 355 // empty string as name is intentional, it would sort before anything else
 356 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
 357                           {"^BM_UserStats/repeats:3 %console_report$"},
 358                           {"^BM_UserStats/repeats:3 %console_report$"},
 359                           {"^BM_UserStats/repeats:3_mean %console_report$"},
 360                           {"^BM_UserStats/repeats:3_median %console_report$"},
 361                           {"^BM_UserStats/repeats:3_stddev %console_report$"},
 362                           {"^BM_UserStats/repeats:3_ %console_report$"}});
 363 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
 364                        {"\"name\": \"BM_UserStats/repeats:3\",$"},
 365                        {"\"name\": \"BM_UserStats/repeats:3\",$"},
 366                        {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
 367                        {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
 368                        {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
 369                        {"\"name\": \"BM_UserStats/repeats:3_\",$"}});
 370 ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
 371                       {"^\"BM_UserStats/repeats:3\",%csv_report$"},
 372                       {"^\"BM_UserStats/repeats:3\",%csv_report$"},
 373                       {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
 374                       {"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
 375                       {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
 376                       {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
 377
 378 // ========================================================================= //
 379 // --------------------------- TEST CASES END ------------------------------ //
 380 // ========================================================================= //
 381
 382 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }