third-party/benchmark/test/reporter_output_test.cc

   1
   2 #undef NDEBUG
   3 #include <numeric>
   4 #include <utility>
   5
   6 #include "benchmark/benchmark.h"
   7 #include "output_test.h"
   8
   9 // ========================================================================= //
  10 // ---------------------- Testing Prologue Output -------------------------- //
  11 // ========================================================================= //
  12
  13 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
  14                           {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
  15                           {"^[-]+$", MR_Next}});
  16 static int AddContextCases() {
  17   AddCases(TC_ConsoleErr,
  18            {
  19                {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
  20                {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next},
  21                {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
  22            });
  23   AddCases(TC_JSONOut,
  24            {{"^\\{", MR_Default},
  25             {"\"context\":", MR_Next},
  26             {"\"date\": \"", MR_Next},
  27             {"\"host_name\":", MR_Next},
  28             {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
  29              MR_Next},
  30             {"\"num_cpus\": %int,$", MR_Next},
  31             {"\"mhz_per_cpu\": %float,$", MR_Next},
  32             {"\"caches\": \\[$", MR_Default}});
  33   auto const& Info = benchmark::CPUInfo::Get();
  34   auto const& Caches = Info.caches;
  35   if (!Caches.empty()) {
  36     AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
  37   }
  38   for (size_t I = 0; I < Caches.size(); ++I) {
  39     std::string num_caches_str =
  40         Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
  41     AddCases(TC_ConsoleErr,
  42              {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
  43                MR_Next}});
  44     AddCases(TC_JSONOut, {{"\\{$", MR_Next},
  45                           {"\"type\": \"", MR_Next},
  46                           {"\"level\": %int,$", MR_Next},
  47                           {"\"size\": %int,$", MR_Next},
  48                           {"\"num_sharing\": %int$", MR_Next},
  49                           {"}[,]{0,1}$", MR_Next}});
  50   }
  51   AddCases(TC_JSONOut, {{"],$"}});
  52   auto const& LoadAvg = Info.load_avg;
  53   if (!LoadAvg.empty()) {
  54     AddCases(TC_ConsoleErr,
  55              {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
  56   }
  57   AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
  58   AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}});
  59   AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}});
  60   AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}});
  61   return 0;
  62 }
  63 int dummy_register = AddContextCases();
  64 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
  65
  66 // ========================================================================= //
  67 // ------------------------ Testing Basic Output --------------------------- //
  68 // ========================================================================= //
  69
  70 void BM_basic(benchmark::State& state) {
  71   for (auto _ : state) {
  72   }
  73 }
  74 BENCHMARK(BM_basic);
  75
  76 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
  77 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
  78                        {"\"family_index\": 0,$", MR_Next},
  79                        {"\"per_family_instance_index\": 0,$", MR_Next},
  80                        {"\"run_name\": \"BM_basic\",$", MR_Next},
  81                        {"\"run_type\": \"iteration\",$", MR_Next},
  82                        {"\"repetitions\": 1,$", MR_Next},
  83                        {"\"repetition_index\": 0,$", MR_Next},
  84                        {"\"threads\": 1,$", MR_Next},
  85                        {"\"iterations\": %int,$", MR_Next},
  86                        {"\"real_time\": %float,$", MR_Next},
  87                        {"\"cpu_time\": %float,$", MR_Next},
  88                        {"\"time_unit\": \"ns\"$", MR_Next},
  89                        {"}", MR_Next}});
  90 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
  91
  92 // ========================================================================= //
  93 // ------------------------ Testing Bytes per Second Output ---------------- //
  94 // ========================================================================= //
  95
  96 void BM_bytes_per_second(benchmark::State& state) {
  97   for (auto _ : state) {
  98     // This test requires a non-zero CPU time to avoid divide-by-zero
  99     auto iterations = double(state.iterations()) * double(state.iterations());
 100     benchmark::DoNotOptimize(iterations);
 101   }
 102   state.SetBytesProcessed(1);
 103 }
 104 BENCHMARK(BM_bytes_per_second);
 105
 106 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
 107                            "bytes_per_second=%float[kM]{0,1}/s$"}});
 108 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
 109                        {"\"family_index\": 1,$", MR_Next},
 110                        {"\"per_family_instance_index\": 0,$", MR_Next},
 111                        {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
 112                        {"\"run_type\": \"iteration\",$", MR_Next},
 113                        {"\"repetitions\": 1,$", MR_Next},
 114                        {"\"repetition_index\": 0,$", MR_Next},
 115                        {"\"threads\": 1,$", MR_Next},
 116                        {"\"iterations\": %int,$", MR_Next},
 117                        {"\"real_time\": %float,$", MR_Next},
 118                        {"\"cpu_time\": %float,$", MR_Next},
 119                        {"\"time_unit\": \"ns\",$", MR_Next},
 120                        {"\"bytes_per_second\": %float$", MR_Next},
 121                        {"}", MR_Next}});
 122 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
 123
 124 // ========================================================================= //
 125 // ------------------------ Testing Items per Second Output ---------------- //
 126 // ========================================================================= //
 127
 128 void BM_items_per_second(benchmark::State& state) {
 129   for (auto _ : state) {
 130     // This test requires a non-zero CPU time to avoid divide-by-zero
 131     auto iterations = double(state.iterations()) * double(state.iterations());
 132     benchmark::DoNotOptimize(iterations);
 133   }
 134   state.SetItemsProcessed(1);
 135 }
 136 BENCHMARK(BM_items_per_second);
 137
 138 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
 139                            "items_per_second=%float[kM]{0,1}/s$"}});
 140 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
 141                        {"\"family_index\": 2,$", MR_Next},
 142                        {"\"per_family_instance_index\": 0,$", MR_Next},
 143                        {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
 144                        {"\"run_type\": \"iteration\",$", MR_Next},
 145                        {"\"repetitions\": 1,$", MR_Next},
 146                        {"\"repetition_index\": 0,$", MR_Next},
 147                        {"\"threads\": 1,$", MR_Next},
 148                        {"\"iterations\": %int,$", MR_Next},
 149                        {"\"real_time\": %float,$", MR_Next},
 150                        {"\"cpu_time\": %float,$", MR_Next},
 151                        {"\"time_unit\": \"ns\",$", MR_Next},
 152                        {"\"items_per_second\": %float$", MR_Next},
 153                        {"}", MR_Next}});
 154 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
 155
 156 // ========================================================================= //
 157 // ------------------------ Testing Label Output --------------------------- //
 158 // ========================================================================= //
 159
 160 void BM_label(benchmark::State& state) {
 161   for (auto _ : state) {
 162   }
 163   state.SetLabel("some label");
 164 }
 165 BENCHMARK(BM_label);
 166
 167 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
 168 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
 169                        {"\"family_index\": 3,$", MR_Next},
 170                        {"\"per_family_instance_index\": 0,$", MR_Next},
 171                        {"\"run_name\": \"BM_label\",$", MR_Next},
 172                        {"\"run_type\": \"iteration\",$", MR_Next},
 173                        {"\"repetitions\": 1,$", MR_Next},
 174                        {"\"repetition_index\": 0,$", MR_Next},
 175                        {"\"threads\": 1,$", MR_Next},
 176                        {"\"iterations\": %int,$", MR_Next},
 177                        {"\"real_time\": %float,$", MR_Next},
 178                        {"\"cpu_time\": %float,$", MR_Next},
 179                        {"\"time_unit\": \"ns\",$", MR_Next},
 180                        {"\"label\": \"some label\"$", MR_Next},
 181                        {"}", MR_Next}});
 182 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
 183                        "label\"%csv_label_report_end$"}});
 184
 185 // ========================================================================= //
 186 // ------------------------ Testing Time Label Output ---------------------- //
 187 // ========================================================================= //
 188
 189 void BM_time_label_nanosecond(benchmark::State& state) {
 190   for (auto _ : state) {
 191   }
 192 }
 193 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
 194
 195 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
 196 ADD_CASES(TC_JSONOut,
 197           {{"\"name\": \"BM_time_label_nanosecond\",$"},
 198            {"\"family_index\": 4,$", MR_Next},
 199            {"\"per_family_instance_index\": 0,$", MR_Next},
 200            {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
 201            {"\"run_type\": \"iteration\",$", MR_Next},
 202            {"\"repetitions\": 1,$", MR_Next},
 203            {"\"repetition_index\": 0,$", MR_Next},
 204            {"\"threads\": 1,$", MR_Next},
 205            {"\"iterations\": %int,$", MR_Next},
 206            {"\"real_time\": %float,$", MR_Next},
 207            {"\"cpu_time\": %float,$", MR_Next},
 208            {"\"time_unit\": \"ns\"$", MR_Next},
 209            {"}", MR_Next}});
 210 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
 211
 212 void BM_time_label_microsecond(benchmark::State& state) {
 213   for (auto _ : state) {
 214   }
 215 }
 216 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
 217
 218 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
 219 ADD_CASES(TC_JSONOut,
 220           {{"\"name\": \"BM_time_label_microsecond\",$"},
 221            {"\"family_index\": 5,$", MR_Next},
 222            {"\"per_family_instance_index\": 0,$", MR_Next},
 223            {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
 224            {"\"run_type\": \"iteration\",$", MR_Next},
 225            {"\"repetitions\": 1,$", MR_Next},
 226            {"\"repetition_index\": 0,$", MR_Next},
 227            {"\"threads\": 1,$", MR_Next},
 228            {"\"iterations\": %int,$", MR_Next},
 229            {"\"real_time\": %float,$", MR_Next},
 230            {"\"cpu_time\": %float,$", MR_Next},
 231            {"\"time_unit\": \"us\"$", MR_Next},
 232            {"}", MR_Next}});
 233 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
 234
 235 void BM_time_label_millisecond(benchmark::State& state) {
 236   for (auto _ : state) {
 237   }
 238 }
 239 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
 240
 241 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
 242 ADD_CASES(TC_JSONOut,
 243           {{"\"name\": \"BM_time_label_millisecond\",$"},
 244            {"\"family_index\": 6,$", MR_Next},
 245            {"\"per_family_instance_index\": 0,$", MR_Next},
 246            {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
 247            {"\"run_type\": \"iteration\",$", MR_Next},
 248            {"\"repetitions\": 1,$", MR_Next},
 249            {"\"repetition_index\": 0,$", MR_Next},
 250            {"\"threads\": 1,$", MR_Next},
 251            {"\"iterations\": %int,$", MR_Next},
 252            {"\"real_time\": %float,$", MR_Next},
 253            {"\"cpu_time\": %float,$", MR_Next},
 254            {"\"time_unit\": \"ms\"$", MR_Next},
 255            {"}", MR_Next}});
 256 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
 257
 258 void BM_time_label_second(benchmark::State& state) {
 259   for (auto _ : state) {
 260   }
 261 }
 262 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
 263
 264 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
 265 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
 266                        {"\"family_index\": 7,$", MR_Next},
 267                        {"\"per_family_instance_index\": 0,$", MR_Next},
 268                        {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
 269                        {"\"run_type\": \"iteration\",$", MR_Next},
 270                        {"\"repetitions\": 1,$", MR_Next},
 271                        {"\"repetition_index\": 0,$", MR_Next},
 272                        {"\"threads\": 1,$", MR_Next},
 273                        {"\"iterations\": %int,$", MR_Next},
 274                        {"\"real_time\": %float,$", MR_Next},
 275                        {"\"cpu_time\": %float,$", MR_Next},
 276                        {"\"time_unit\": \"s\"$", MR_Next},
 277                        {"}", MR_Next}});
 278 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
 279
 280 // ========================================================================= //
 281 // ------------------------ Testing Error Output --------------------------- //
 282 // ========================================================================= //
 283
 284 void BM_error(benchmark::State& state) {
 285   state.SkipWithError("message");
 286   for (auto _ : state) {
 287   }
 288 }
 289 BENCHMARK(BM_error);
 290 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
 291 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
 292                        {"\"family_index\": 8,$", MR_Next},
 293                        {"\"per_family_instance_index\": 0,$", MR_Next},
 294                        {"\"run_name\": \"BM_error\",$", MR_Next},
 295                        {"\"run_type\": \"iteration\",$", MR_Next},
 296                        {"\"repetitions\": 1,$", MR_Next},
 297                        {"\"repetition_index\": 0,$", MR_Next},
 298                        {"\"threads\": 1,$", MR_Next},
 299                        {"\"error_occurred\": true,$", MR_Next},
 300                        {"\"error_message\": \"message\",$", MR_Next}});
 301
 302 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
 303
 304 // ========================================================================= //
 305 // ------------------------ Testing No Arg Name Output -----------------------
 306 // //
 307 // ========================================================================= //
 308
 309 void BM_no_arg_name(benchmark::State& state) {
 310   for (auto _ : state) {
 311   }
 312 }
 313 BENCHMARK(BM_no_arg_name)->Arg(3);
 314 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
 315 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
 316                        {"\"family_index\": 9,$", MR_Next},
 317                        {"\"per_family_instance_index\": 0,$", MR_Next},
 318                        {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
 319                        {"\"run_type\": \"iteration\",$", MR_Next},
 320                        {"\"repetitions\": 1,$", MR_Next},
 321                        {"\"repetition_index\": 0,$", MR_Next},
 322                        {"\"threads\": 1,$", MR_Next}});
 323 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
 324
 325 // ========================================================================= //
 326 // ------------------------ Testing Arg Name Output ------------------------ //
 327 // ========================================================================= //
 328
 329 void BM_arg_name(benchmark::State& state) {
 330   for (auto _ : state) {
 331   }
 332 }
 333 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
 334 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
 335 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
 336                        {"\"family_index\": 10,$", MR_Next},
 337                        {"\"per_family_instance_index\": 0,$", MR_Next},
 338                        {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
 339                        {"\"run_type\": \"iteration\",$", MR_Next},
 340                        {"\"repetitions\": 1,$", MR_Next},
 341                        {"\"repetition_index\": 0,$", MR_Next},
 342                        {"\"threads\": 1,$", MR_Next}});
 343 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
 344
 345 // ========================================================================= //
 346 // ------------------------ Testing Arg Names Output ----------------------- //
 347 // ========================================================================= //
 348
 349 void BM_arg_names(benchmark::State& state) {
 350   for (auto _ : state) {
 351   }
 352 }
 353 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
 354 ADD_CASES(TC_ConsoleOut,
 355           {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
 356 ADD_CASES(TC_JSONOut,
 357           {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
 358            {"\"family_index\": 11,$", MR_Next},
 359            {"\"per_family_instance_index\": 0,$", MR_Next},
 360            {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
 361            {"\"run_type\": \"iteration\",$", MR_Next},
 362            {"\"repetitions\": 1,$", MR_Next},
 363            {"\"repetition_index\": 0,$", MR_Next},
 364            {"\"threads\": 1,$", MR_Next}});
 365 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
 366
 367 // ========================================================================= //
 368 // ------------------------ Testing Name Output ---------------------------- //
 369 // ========================================================================= //
 370
 371 void BM_name(benchmark::State& state) {
 372   for (auto _ : state) {
 373   }
 374 }
 375 BENCHMARK(BM_name)->Name("BM_custom_name");
 376
 377 ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
 378 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
 379                        {"\"family_index\": 12,$", MR_Next},
 380                        {"\"per_family_instance_index\": 0,$", MR_Next},
 381                        {"\"run_name\": \"BM_custom_name\",$", MR_Next},
 382                        {"\"run_type\": \"iteration\",$", MR_Next},
 383                        {"\"repetitions\": 1,$", MR_Next},
 384                        {"\"repetition_index\": 0,$", MR_Next},
 385                        {"\"threads\": 1,$", MR_Next},
 386                        {"\"iterations\": %int,$", MR_Next},
 387                        {"\"real_time\": %float,$", MR_Next},
 388                        {"\"cpu_time\": %float,$", MR_Next},
 389                        {"\"time_unit\": \"ns\"$", MR_Next},
 390                        {"}", MR_Next}});
 391 ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
 392
 393 // ========================================================================= //
 394 // ------------------------ Testing Big Args Output ------------------------ //
 395 // ========================================================================= //
 396
 397 void BM_BigArgs(benchmark::State& state) {
 398   for (auto _ : state) {
 399   }
 400 }
 401 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
 402 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
 403                           {"^BM_BigArgs/2147483648 %console_report$"}});
 404
 405 // ========================================================================= //
 406 // ----------------------- Testing Complexity Output ----------------------- //
 407 // ========================================================================= //
 408
 409 void BM_Complexity_O1(benchmark::State& state) {
 410   for (auto _ : state) {
 411     // This test requires a non-zero CPU time to avoid divide-by-zero
 412     auto iterations = double(state.iterations()) * double(state.iterations());
 413     benchmark::DoNotOptimize(iterations);
 414   }
 415   state.SetComplexityN(state.range(0));
 416 }
 417 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
 418 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
 419                    {"%RMS", "[ ]*[0-9]+ %"}});
 420 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
 421                           {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
 422
 423 // ========================================================================= //
 424 // ----------------------- Testing Aggregate Output ------------------------ //
 425 // ========================================================================= //
 426
 427 // Test that non-aggregate data is printed by default
 428 void BM_Repeat(benchmark::State& state) {
 429   for (auto _ : state) {
 430   }
 431 }
 432 // need two repetitions min to be able to output any aggregate output
 433 BENCHMARK(BM_Repeat)->Repetitions(2);
 434 ADD_CASES(TC_ConsoleOut,
 435           {{"^BM_Repeat/repeats:2 %console_report$"},
 436            {"^BM_Repeat/repeats:2 %console_report$"},
 437            {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
 438            {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
 439            {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
 440 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
 441                        {"\"family_index\": 15,$", MR_Next},
 442                        {"\"per_family_instance_index\": 0,$", MR_Next},
 443                        {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
 444                        {"\"run_type\": \"iteration\",$", MR_Next},
 445                        {"\"repetitions\": 2,$", MR_Next},
 446                        {"\"repetition_index\": 0,$", MR_Next},
 447                        {"\"threads\": 1,$", MR_Next},
 448                        {"\"name\": \"BM_Repeat/repeats:2\",$"},
 449                        {"\"family_index\": 15,$", MR_Next},
 450                        {"\"per_family_instance_index\": 0,$", MR_Next},
 451                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 452                        {"\"run_type\": \"iteration\",$", MR_Next},
 453                        {"\"repetitions\": 2,$", MR_Next},
 454                        {"\"repetition_index\": 1,$", MR_Next},
 455                        {"\"threads\": 1,$", MR_Next},
 456                        {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
 457                        {"\"family_index\": 15,$", MR_Next},
 458                        {"\"per_family_instance_index\": 0,$", MR_Next},
 459                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 460                        {"\"run_type\": \"aggregate\",$", MR_Next},
 461                        {"\"repetitions\": 2,$", MR_Next},
 462                        {"\"threads\": 1,$", MR_Next},
 463                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 464                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 465                        {"\"iterations\": 2,$", MR_Next},
 466                        {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
 467                        {"\"family_index\": 15,$", MR_Next},
 468                        {"\"per_family_instance_index\": 0,$", MR_Next},
 469                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 470                        {"\"run_type\": \"aggregate\",$", MR_Next},
 471                        {"\"repetitions\": 2,$", MR_Next},
 472                        {"\"threads\": 1,$", MR_Next},
 473                        {"\"aggregate_name\": \"median\",$", MR_Next},
 474                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 475                        {"\"iterations\": 2,$", MR_Next},
 476                        {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
 477                        {"\"family_index\": 15,$", MR_Next},
 478                        {"\"per_family_instance_index\": 0,$", MR_Next},
 479                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 480                        {"\"run_type\": \"aggregate\",$", MR_Next},
 481                        {"\"repetitions\": 2,$", MR_Next},
 482                        {"\"threads\": 1,$", MR_Next},
 483                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 484                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 485                        {"\"iterations\": 2,$", MR_Next}});
 486 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
 487                       {"^\"BM_Repeat/repeats:2\",%csv_report$"},
 488                       {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
 489                       {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
 490                       {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
 491 // but for two repetitions, mean and median is the same, so let's repeat..
 492 BENCHMARK(BM_Repeat)->Repetitions(3);
 493 ADD_CASES(TC_ConsoleOut,
 494           {{"^BM_Repeat/repeats:3 %console_report$"},
 495            {"^BM_Repeat/repeats:3 %console_report$"},
 496            {"^BM_Repeat/repeats:3 %console_report$"},
 497            {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
 498            {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
 499            {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
 500 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
 501                        {"\"family_index\": 16,$", MR_Next},
 502                        {"\"per_family_instance_index\": 0,$", MR_Next},
 503                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 504                        {"\"run_type\": \"iteration\",$", MR_Next},
 505                        {"\"repetitions\": 3,$", MR_Next},
 506                        {"\"repetition_index\": 0,$", MR_Next},
 507                        {"\"threads\": 1,$", MR_Next},
 508                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 509                        {"\"family_index\": 16,$", MR_Next},
 510                        {"\"per_family_instance_index\": 0,$", MR_Next},
 511                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 512                        {"\"run_type\": \"iteration\",$", MR_Next},
 513                        {"\"repetitions\": 3,$", MR_Next},
 514                        {"\"repetition_index\": 1,$", MR_Next},
 515                        {"\"threads\": 1,$", MR_Next},
 516                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 517                        {"\"family_index\": 16,$", MR_Next},
 518                        {"\"per_family_instance_index\": 0,$", MR_Next},
 519                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 520                        {"\"run_type\": \"iteration\",$", MR_Next},
 521                        {"\"repetitions\": 3,$", MR_Next},
 522                        {"\"repetition_index\": 2,$", MR_Next},
 523                        {"\"threads\": 1,$", MR_Next},
 524                        {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
 525                        {"\"family_index\": 16,$", MR_Next},
 526                        {"\"per_family_instance_index\": 0,$", MR_Next},
 527                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 528                        {"\"run_type\": \"aggregate\",$", MR_Next},
 529                        {"\"repetitions\": 3,$", MR_Next},
 530                        {"\"threads\": 1,$", MR_Next},
 531                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 532                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 533                        {"\"iterations\": 3,$", MR_Next},
 534                        {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
 535                        {"\"family_index\": 16,$", MR_Next},
 536                        {"\"per_family_instance_index\": 0,$", MR_Next},
 537                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 538                        {"\"run_type\": \"aggregate\",$", MR_Next},
 539                        {"\"repetitions\": 3,$", MR_Next},
 540                        {"\"threads\": 1,$", MR_Next},
 541                        {"\"aggregate_name\": \"median\",$", MR_Next},
 542                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 543                        {"\"iterations\": 3,$", MR_Next},
 544                        {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
 545                        {"\"family_index\": 16,$", MR_Next},
 546                        {"\"per_family_instance_index\": 0,$", MR_Next},
 547                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 548                        {"\"run_type\": \"aggregate\",$", MR_Next},
 549                        {"\"repetitions\": 3,$", MR_Next},
 550                        {"\"threads\": 1,$", MR_Next},
 551                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 552                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 553                        {"\"iterations\": 3,$", MR_Next}});
 554 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
 555                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 556                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 557                       {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
 558                       {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
 559                       {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
 560 // median differs between even/odd number of repetitions, so just to be sure
 561 BENCHMARK(BM_Repeat)->Repetitions(4);
 562 ADD_CASES(TC_ConsoleOut,
 563           {{"^BM_Repeat/repeats:4 %console_report$"},
 564            {"^BM_Repeat/repeats:4 %console_report$"},
 565            {"^BM_Repeat/repeats:4 %console_report$"},
 566            {"^BM_Repeat/repeats:4 %console_report$"},
 567            {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
 568            {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
 569            {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
 570 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
 571                        {"\"family_index\": 17,$", MR_Next},
 572                        {"\"per_family_instance_index\": 0,$", MR_Next},
 573                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 574                        {"\"run_type\": \"iteration\",$", MR_Next},
 575                        {"\"repetitions\": 4,$", MR_Next},
 576                        {"\"repetition_index\": 0,$", MR_Next},
 577                        {"\"threads\": 1,$", MR_Next},
 578                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 579                        {"\"family_index\": 17,$", MR_Next},
 580                        {"\"per_family_instance_index\": 0,$", MR_Next},
 581                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 582                        {"\"run_type\": \"iteration\",$", MR_Next},
 583                        {"\"repetitions\": 4,$", MR_Next},
 584                        {"\"repetition_index\": 1,$", MR_Next},
 585                        {"\"threads\": 1,$", MR_Next},
 586                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 587                        {"\"family_index\": 17,$", MR_Next},
 588                        {"\"per_family_instance_index\": 0,$", MR_Next},
 589                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 590                        {"\"run_type\": \"iteration\",$", MR_Next},
 591                        {"\"repetitions\": 4,$", MR_Next},
 592                        {"\"repetition_index\": 2,$", MR_Next},
 593                        {"\"threads\": 1,$", MR_Next},
 594                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 595                        {"\"family_index\": 17,$", MR_Next},
 596                        {"\"per_family_instance_index\": 0,$", MR_Next},
 597                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 598                        {"\"run_type\": \"iteration\",$", MR_Next},
 599                        {"\"repetitions\": 4,$", MR_Next},
 600                        {"\"repetition_index\": 3,$", MR_Next},
 601                        {"\"threads\": 1,$", MR_Next},
 602                        {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
 603                        {"\"family_index\": 17,$", MR_Next},
 604                        {"\"per_family_instance_index\": 0,$", MR_Next},
 605                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 606                        {"\"run_type\": \"aggregate\",$", MR_Next},
 607                        {"\"repetitions\": 4,$", MR_Next},
 608                        {"\"threads\": 1,$", MR_Next},
 609                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 610                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 611                        {"\"iterations\": 4,$", MR_Next},
 612                        {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
 613                        {"\"family_index\": 17,$", MR_Next},
 614                        {"\"per_family_instance_index\": 0,$", MR_Next},
 615                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 616                        {"\"run_type\": \"aggregate\",$", MR_Next},
 617                        {"\"repetitions\": 4,$", MR_Next},
 618                        {"\"threads\": 1,$", MR_Next},
 619                        {"\"aggregate_name\": \"median\",$", MR_Next},
 620                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 621                        {"\"iterations\": 4,$", MR_Next},
 622                        {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
 623                        {"\"family_index\": 17,$", MR_Next},
 624                        {"\"per_family_instance_index\": 0,$", MR_Next},
 625                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 626                        {"\"run_type\": \"aggregate\",$", MR_Next},
 627                        {"\"repetitions\": 4,$", MR_Next},
 628                        {"\"threads\": 1,$", MR_Next},
 629                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 630                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 631                        {"\"iterations\": 4,$", MR_Next}});
 632 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
 633                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 634                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 635                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 636                       {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
 637                       {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
 638                       {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
 639
 640 // Test that a non-repeated test still prints non-aggregate results even when
 641 // only-aggregate reports have been requested
 642 void BM_RepeatOnce(benchmark::State& state) {
 643   for (auto _ : state) {
 644   }
 645 }
 646 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
 647 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
 648 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
 649                        {"\"family_index\": 18,$", MR_Next},
 650                        {"\"per_family_instance_index\": 0,$", MR_Next},
 651                        {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
 652                        {"\"run_type\": \"iteration\",$", MR_Next},
 653                        {"\"repetitions\": 1,$", MR_Next},
 654                        {"\"repetition_index\": 0,$", MR_Next},
 655                        {"\"threads\": 1,$", MR_Next}});
 656 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
 657
 658 // Test that non-aggregate data is not reported
 659 void BM_SummaryRepeat(benchmark::State& state) {
 660   for (auto _ : state) {
 661   }
 662 }
 663 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
 664 ADD_CASES(
 665     TC_ConsoleOut,
 666     {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 667      {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
 668      {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
 669      {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
 670 ADD_CASES(TC_JSONOut,
 671           {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 672            {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
 673            {"\"family_index\": 19,$", MR_Next},
 674            {"\"per_family_instance_index\": 0,$", MR_Next},
 675            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 676            {"\"run_type\": \"aggregate\",$", MR_Next},
 677            {"\"repetitions\": 3,$", MR_Next},
 678            {"\"threads\": 1,$", MR_Next},
 679            {"\"aggregate_name\": \"mean\",$", MR_Next},
 680            {"\"aggregate_unit\": \"time\",$", MR_Next},
 681            {"\"iterations\": 3,$", MR_Next},
 682            {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
 683            {"\"family_index\": 19,$", MR_Next},
 684            {"\"per_family_instance_index\": 0,$", MR_Next},
 685            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 686            {"\"run_type\": \"aggregate\",$", MR_Next},
 687            {"\"repetitions\": 3,$", MR_Next},
 688            {"\"threads\": 1,$", MR_Next},
 689            {"\"aggregate_name\": \"median\",$", MR_Next},
 690            {"\"aggregate_unit\": \"time\",$", MR_Next},
 691            {"\"iterations\": 3,$", MR_Next},
 692            {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
 693            {"\"family_index\": 19,$", MR_Next},
 694            {"\"per_family_instance_index\": 0,$", MR_Next},
 695            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 696            {"\"run_type\": \"aggregate\",$", MR_Next},
 697            {"\"repetitions\": 3,$", MR_Next},
 698            {"\"threads\": 1,$", MR_Next},
 699            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 700            {"\"aggregate_unit\": \"time\",$", MR_Next},
 701            {"\"iterations\": 3,$", MR_Next}});
 702 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 703                       {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
 704                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
 705                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
 706
 707 // Test that non-aggregate data is not displayed.
 708 // NOTE: this test is kinda bad. we are only testing the display output.
 709 //       But we don't check that the file output still contains everything...
 710 void BM_SummaryDisplay(benchmark::State& state) {
 711   for (auto _ : state) {
 712   }
 713 }
 714 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
 715 ADD_CASES(
 716     TC_ConsoleOut,
 717     {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 718      {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
 719      {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
 720      {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
 721 ADD_CASES(TC_JSONOut,
 722           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 723            {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
 724            {"\"family_index\": 20,$", MR_Next},
 725            {"\"per_family_instance_index\": 0,$", MR_Next},
 726            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 727            {"\"run_type\": \"aggregate\",$", MR_Next},
 728            {"\"repetitions\": 2,$", MR_Next},
 729            {"\"threads\": 1,$", MR_Next},
 730            {"\"aggregate_name\": \"mean\",$", MR_Next},
 731            {"\"aggregate_unit\": \"time\",$", MR_Next},
 732            {"\"iterations\": 2,$", MR_Next},
 733            {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
 734            {"\"family_index\": 20,$", MR_Next},
 735            {"\"per_family_instance_index\": 0,$", MR_Next},
 736            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 737            {"\"run_type\": \"aggregate\",$", MR_Next},
 738            {"\"repetitions\": 2,$", MR_Next},
 739            {"\"threads\": 1,$", MR_Next},
 740            {"\"aggregate_name\": \"median\",$", MR_Next},
 741            {"\"aggregate_unit\": \"time\",$", MR_Next},
 742            {"\"iterations\": 2,$", MR_Next},
 743            {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
 744            {"\"family_index\": 20,$", MR_Next},
 745            {"\"per_family_instance_index\": 0,$", MR_Next},
 746            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 747            {"\"run_type\": \"aggregate\",$", MR_Next},
 748            {"\"repetitions\": 2,$", MR_Next},
 749            {"\"threads\": 1,$", MR_Next},
 750            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 751            {"\"aggregate_unit\": \"time\",$", MR_Next},
 752            {"\"iterations\": 2,$", MR_Next}});
 753 ADD_CASES(TC_CSVOut,
 754           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 755            {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
 756            {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
 757            {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
 758
 759 // Test repeats with custom time unit.
 760 void BM_RepeatTimeUnit(benchmark::State& state) {
 761   for (auto _ : state) {
 762   }
 763 }
 764 BENCHMARK(BM_RepeatTimeUnit)
 765     ->Repetitions(3)
 766     ->ReportAggregatesOnly()
 767     ->Unit(benchmark::kMicrosecond);
 768 ADD_CASES(
 769     TC_ConsoleOut,
 770     {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 771      {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
 772      {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
 773       "]*3$"},
 774      {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
 775       "]*3$"}});
 776 ADD_CASES(TC_JSONOut,
 777           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 778            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
 779            {"\"family_index\": 21,$", MR_Next},
 780            {"\"per_family_instance_index\": 0,$", MR_Next},
 781            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 782            {"\"run_type\": \"aggregate\",$", MR_Next},
 783            {"\"repetitions\": 3,$", MR_Next},
 784            {"\"threads\": 1,$", MR_Next},
 785            {"\"aggregate_name\": \"mean\",$", MR_Next},
 786            {"\"aggregate_unit\": \"time\",$", MR_Next},
 787            {"\"iterations\": 3,$", MR_Next},
 788            {"\"time_unit\": \"us\",?$"},
 789            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
 790            {"\"family_index\": 21,$", MR_Next},
 791            {"\"per_family_instance_index\": 0,$", MR_Next},
 792            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 793            {"\"run_type\": \"aggregate\",$", MR_Next},
 794            {"\"repetitions\": 3,$", MR_Next},
 795            {"\"threads\": 1,$", MR_Next},
 796            {"\"aggregate_name\": \"median\",$", MR_Next},
 797            {"\"aggregate_unit\": \"time\",$", MR_Next},
 798            {"\"iterations\": 3,$", MR_Next},
 799            {"\"time_unit\": \"us\",?$"},
 800            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
 801            {"\"family_index\": 21,$", MR_Next},
 802            {"\"per_family_instance_index\": 0,$", MR_Next},
 803            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 804            {"\"run_type\": \"aggregate\",$", MR_Next},
 805            {"\"repetitions\": 3,$", MR_Next},
 806            {"\"threads\": 1,$", MR_Next},
 807            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 808            {"\"aggregate_unit\": \"time\",$", MR_Next},
 809            {"\"iterations\": 3,$", MR_Next},
 810            {"\"time_unit\": \"us\",?$"}});
 811 ADD_CASES(TC_CSVOut,
 812           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 813            {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
 814            {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
 815            {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
 816
 817 // ========================================================================= //
 818 // -------------------- Testing user-provided statistics ------------------- //
 819 // ========================================================================= //
 820
 821 const auto UserStatistics = [](const std::vector<double>& v) {
 822   return v.back();
 823 };
 824 void BM_UserStats(benchmark::State& state) {
 825   for (auto _ : state) {
 826     state.SetIterationTime(150 / 10e8);
 827   }
 828 }
 829 // clang-format off
 830 BENCHMARK(BM_UserStats)
 831   ->Repetitions(3)
 832   ->Iterations(5)
 833   ->UseManualTime()
 834   ->ComputeStatistics("", UserStatistics);
 835 // clang-format on
 836
 837 // check that user-provided stats is calculated, and is after the default-ones
 838 // empty string as name is intentional, it would sort before anything else
 839 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 840                            "]* 150 ns %time [ ]*5$"},
 841                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 842                            "]* 150 ns %time [ ]*5$"},
 843                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 844                            "]* 150 ns %time [ ]*5$"},
 845                           {"^BM_UserStats/iterations:5/repeats:3/"
 846                            "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
 847                           {"^BM_UserStats/iterations:5/repeats:3/"
 848                            "manual_time_median [ ]* 150 ns %time [ ]*3$"},
 849                           {"^BM_UserStats/iterations:5/repeats:3/"
 850                            "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
 851                           {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
 852                            "[ ]* 150 ns %time [ ]*3$"}});
 853 ADD_CASES(
 854     TC_JSONOut,
 855     {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 856      {"\"family_index\": 22,$", MR_Next},
 857      {"\"per_family_instance_index\": 0,$", MR_Next},
 858      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 859       MR_Next},
 860      {"\"run_type\": \"iteration\",$", MR_Next},
 861      {"\"repetitions\": 3,$", MR_Next},
 862      {"\"repetition_index\": 0,$", MR_Next},
 863      {"\"threads\": 1,$", MR_Next},
 864      {"\"iterations\": 5,$", MR_Next},
 865      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 866      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 867      {"\"family_index\": 22,$", MR_Next},
 868      {"\"per_family_instance_index\": 0,$", MR_Next},
 869      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 870       MR_Next},
 871      {"\"run_type\": \"iteration\",$", MR_Next},
 872      {"\"repetitions\": 3,$", MR_Next},
 873      {"\"repetition_index\": 1,$", MR_Next},
 874      {"\"threads\": 1,$", MR_Next},
 875      {"\"iterations\": 5,$", MR_Next},
 876      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 877      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 878      {"\"family_index\": 22,$", MR_Next},
 879      {"\"per_family_instance_index\": 0,$", MR_Next},
 880      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 881       MR_Next},
 882      {"\"run_type\": \"iteration\",$", MR_Next},
 883      {"\"repetitions\": 3,$", MR_Next},
 884      {"\"repetition_index\": 2,$", MR_Next},
 885      {"\"threads\": 1,$", MR_Next},
 886      {"\"iterations\": 5,$", MR_Next},
 887      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 888      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
 889      {"\"family_index\": 22,$", MR_Next},
 890      {"\"per_family_instance_index\": 0,$", MR_Next},
 891      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 892       MR_Next},
 893      {"\"run_type\": \"aggregate\",$", MR_Next},
 894      {"\"repetitions\": 3,$", MR_Next},
 895      {"\"threads\": 1,$", MR_Next},
 896      {"\"aggregate_name\": \"mean\",$", MR_Next},
 897      {"\"aggregate_unit\": \"time\",$", MR_Next},
 898      {"\"iterations\": 3,$", MR_Next},
 899      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 900      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
 901      {"\"family_index\": 22,$", MR_Next},
 902      {"\"per_family_instance_index\": 0,$", MR_Next},
 903      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 904       MR_Next},
 905      {"\"run_type\": \"aggregate\",$", MR_Next},
 906      {"\"repetitions\": 3,$", MR_Next},
 907      {"\"threads\": 1,$", MR_Next},
 908      {"\"aggregate_name\": \"median\",$", MR_Next},
 909      {"\"aggregate_unit\": \"time\",$", MR_Next},
 910      {"\"iterations\": 3,$", MR_Next},
 911      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 912      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
 913      {"\"family_index\": 22,$", MR_Next},
 914      {"\"per_family_instance_index\": 0,$", MR_Next},
 915      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 916       MR_Next},
 917      {"\"run_type\": \"aggregate\",$", MR_Next},
 918      {"\"repetitions\": 3,$", MR_Next},
 919      {"\"threads\": 1,$", MR_Next},
 920      {"\"aggregate_name\": \"stddev\",$", MR_Next},
 921      {"\"aggregate_unit\": \"time\",$", MR_Next},
 922      {"\"iterations\": 3,$", MR_Next},
 923      {"\"real_time\": %float,$", MR_Next},
 924      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
 925      {"\"family_index\": 22,$", MR_Next},
 926      {"\"per_family_instance_index\": 0,$", MR_Next},
 927      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 928       MR_Next},
 929      {"\"run_type\": \"aggregate\",$", MR_Next},
 930      {"\"repetitions\": 3,$", MR_Next},
 931      {"\"threads\": 1,$", MR_Next},
 932      {"\"aggregate_name\": \"\",$", MR_Next},
 933      {"\"aggregate_unit\": \"time\",$", MR_Next},
 934      {"\"iterations\": 3,$", MR_Next},
 935      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
 936 ADD_CASES(
 937     TC_CSVOut,
 938     {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 939      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 940      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 941      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
 942      {"^\"BM_UserStats/iterations:5/repeats:3/"
 943       "manual_time_median\",%csv_report$"},
 944      {"^\"BM_UserStats/iterations:5/repeats:3/"
 945       "manual_time_stddev\",%csv_report$"},
 946      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
 947
 948 // ========================================================================= //
 949 // ------------- Testing relative standard deviation statistics ------------ //
 950 // ========================================================================= //
 951
 952 const auto UserPercentStatistics = [](const std::vector<double>&) {
 953   return 1. / 100.;
 954 };
 955 void BM_UserPercentStats(benchmark::State& state) {
 956   for (auto _ : state) {
 957     state.SetIterationTime(150 / 10e8);
 958   }
 959 }
 960 // clang-format off
 961 BENCHMARK(BM_UserPercentStats)
 962   ->Repetitions(3)
 963   ->Iterations(5)
 964   ->UseManualTime()
 965   ->Unit(benchmark::TimeUnit::kNanosecond)
 966   ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
 967 // clang-format on
 968
 969 // check that UserPercent-provided stats is calculated, and is after the
 970 // default-ones empty string as name is intentional, it would sort before
 971 // anything else
 972 ADD_CASES(TC_ConsoleOut,
 973           {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 974             "]* 150 ns %time [ ]*5$"},
 975            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 976             "]* 150 ns %time [ ]*5$"},
 977            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 978             "]* 150 ns %time [ ]*5$"},
 979            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 980             "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
 981            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 982             "manual_time_median [ ]* 150 ns %time [ ]*3$"},
 983            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 984             "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
 985            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
 986             "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
 987 ADD_CASES(
 988     TC_JSONOut,
 989     {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
 990      {"\"family_index\": 23,$", MR_Next},
 991      {"\"per_family_instance_index\": 0,$", MR_Next},
 992      {"\"run_name\": "
 993       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
 994       MR_Next},
 995      {"\"run_type\": \"iteration\",$", MR_Next},
 996      {"\"repetitions\": 3,$", MR_Next},
 997      {"\"repetition_index\": 0,$", MR_Next},
 998      {"\"threads\": 1,$", MR_Next},
 999      {"\"iterations\": 5,$", MR_Next},
1000      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1001      {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1002      {"\"family_index\": 23,$", MR_Next},
1003      {"\"per_family_instance_index\": 0,$", MR_Next},
1004      {"\"run_name\": "
1005       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1006       MR_Next},
1007      {"\"run_type\": \"iteration\",$", MR_Next},
1008      {"\"repetitions\": 3,$", MR_Next},
1009      {"\"repetition_index\": 1,$", MR_Next},
1010      {"\"threads\": 1,$", MR_Next},
1011      {"\"iterations\": 5,$", MR_Next},
1012      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1013      {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1014      {"\"family_index\": 23,$", MR_Next},
1015      {"\"per_family_instance_index\": 0,$", MR_Next},
1016      {"\"run_name\": "
1017       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1018       MR_Next},
1019      {"\"run_type\": \"iteration\",$", MR_Next},
1020      {"\"repetitions\": 3,$", MR_Next},
1021      {"\"repetition_index\": 2,$", MR_Next},
1022      {"\"threads\": 1,$", MR_Next},
1023      {"\"iterations\": 5,$", MR_Next},
1024      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1025      {"\"name\": "
1026       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1027      {"\"family_index\": 23,$", MR_Next},
1028      {"\"per_family_instance_index\": 0,$", MR_Next},
1029      {"\"run_name\": "
1030       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1031       MR_Next},
1032      {"\"run_type\": \"aggregate\",$", MR_Next},
1033      {"\"repetitions\": 3,$", MR_Next},
1034      {"\"threads\": 1,$", MR_Next},
1035      {"\"aggregate_name\": \"mean\",$", MR_Next},
1036      {"\"aggregate_unit\": \"time\",$", MR_Next},
1037      {"\"iterations\": 3,$", MR_Next},
1038      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1039      {"\"name\": "
1040       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1041      {"\"family_index\": 23,$", MR_Next},
1042      {"\"per_family_instance_index\": 0,$", MR_Next},
1043      {"\"run_name\": "
1044       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1045       MR_Next},
1046      {"\"run_type\": \"aggregate\",$", MR_Next},
1047      {"\"repetitions\": 3,$", MR_Next},
1048      {"\"threads\": 1,$", MR_Next},
1049      {"\"aggregate_name\": \"median\",$", MR_Next},
1050      {"\"aggregate_unit\": \"time\",$", MR_Next},
1051      {"\"iterations\": 3,$", MR_Next},
1052      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1053      {"\"name\": "
1054       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1055      {"\"family_index\": 23,$", MR_Next},
1056      {"\"per_family_instance_index\": 0,$", MR_Next},
1057      {"\"run_name\": "
1058       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1059       MR_Next},
1060      {"\"run_type\": \"aggregate\",$", MR_Next},
1061      {"\"repetitions\": 3,$", MR_Next},
1062      {"\"threads\": 1,$", MR_Next},
1063      {"\"aggregate_name\": \"stddev\",$", MR_Next},
1064      {"\"aggregate_unit\": \"time\",$", MR_Next},
1065      {"\"iterations\": 3,$", MR_Next},
1066      {"\"real_time\": %float,$", MR_Next},
1067      {"\"name\": "
1068       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1069      {"\"family_index\": 23,$", MR_Next},
1070      {"\"per_family_instance_index\": 0,$", MR_Next},
1071      {"\"run_name\": "
1072       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1073       MR_Next},
1074      {"\"run_type\": \"aggregate\",$", MR_Next},
1075      {"\"repetitions\": 3,$", MR_Next},
1076      {"\"threads\": 1,$", MR_Next},
1077      {"\"aggregate_name\": \"\",$", MR_Next},
1078      {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1079      {"\"iterations\": 3,$", MR_Next},
1080      {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1081 ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1082                        "manual_time\",%csv_report$"},
1083                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1084                        "manual_time\",%csv_report$"},
1085                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1086                        "manual_time\",%csv_report$"},
1087                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1088                        "manual_time_mean\",%csv_report$"},
1089                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1090                        "manual_time_median\",%csv_report$"},
1091                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1092                        "manual_time_stddev\",%csv_report$"},
1093                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1094                        "manual_time_\",%csv_cv_report$"}});
1095
1096 // ========================================================================= //
1097 // ------------------------- Testing StrEscape JSON ------------------------ //
1098 // ========================================================================= //
1099 #if 0  // enable when csv testing code correctly handles multi-line fields
1100 void BM_JSON_Format(benchmark::State& state) {
1101   state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1102   for (auto _ : state) {
1103   }
1104 }
1105 BENCHMARK(BM_JSON_Format);
1106 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1107                                               {"\"family_index\": 23,$", MR_Next},
1108 {"\"per_family_instance_index\": 0,$", MR_Next},
1109                        {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1110                        {"\"run_type\": \"iteration\",$", MR_Next},
1111                        {"\"repetitions\": 1,$", MR_Next},
1112                        {"\"repetition_index\": 0,$", MR_Next},
1113                        {"\"threads\": 1,$", MR_Next},
1114                        {"\"error_occurred\": true,$", MR_Next},
1115                        {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1116 #endif
1117 // ========================================================================= //
1118 // -------------------------- Testing CsvEscape ---------------------------- //
1119 // ========================================================================= //
1120
1121 void BM_CSV_Format(benchmark::State& state) {
1122   state.SkipWithError("\"freedom\"");
1123   for (auto _ : state) {
1124   }
1125 }
1126 BENCHMARK(BM_CSV_Format);
1127 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1128
1129 // ========================================================================= //
1130 // --------------------------- TEST CASES END ------------------------------ //
1131 // ========================================================================= //
1132
1133 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }