third-party/benchmark/test/reporter_output_test.cc

   1
   2 #undef NDEBUG
   3 #include <numeric>
   4 #include <utility>
   5
   6 #include "benchmark/benchmark.h"
   7 #include "output_test.h"
   8
   9 // ========================================================================= //
  10 // ---------------------- Testing Prologue Output -------------------------- //
  11 // ========================================================================= //
  12
  13 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
  14                           {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
  15                           {"^[-]+$", MR_Next}});
  16 static int AddContextCases() {
  17   AddCases(TC_ConsoleErr,
  18            {
  19                {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
  20                {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
  21                {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
  22            });
  23   AddCases(TC_JSONOut,
  24            {{"^\\{", MR_Default},
  25             {"\"context\":", MR_Next},
  26             {"\"date\": \"", MR_Next},
  27             {"\"host_name\":", MR_Next},
  28             {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
  29              MR_Next},
  30             {"\"num_cpus\": %int,$", MR_Next},
  31             {"\"mhz_per_cpu\": %float,$", MR_Next},
  32             {"\"caches\": \\[$", MR_Default}});
  33   auto const& Info = benchmark::CPUInfo::Get();
  34   auto const& Caches = Info.caches;
  35   if (!Caches.empty()) {
  36     AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
  37   }
  38   for (size_t I = 0; I < Caches.size(); ++I) {
  39     std::string num_caches_str =
  40         Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
  41     AddCases(TC_ConsoleErr,
  42              {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
  43                MR_Next}});
  44     AddCases(TC_JSONOut, {{"\\{$", MR_Next},
  45                           {"\"type\": \"", MR_Next},
  46                           {"\"level\": %int,$", MR_Next},
  47                           {"\"size\": %int,$", MR_Next},
  48                           {"\"num_sharing\": %int$", MR_Next},
  49                           {"}[,]{0,1}$", MR_Next}});
  50   }
  51   AddCases(TC_JSONOut, {{"],$"}});
  52   auto const& LoadAvg = Info.load_avg;
  53   if (!LoadAvg.empty()) {
  54     AddCases(TC_ConsoleErr,
  55              {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
  56   }
  57   AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
  58   return 0;
  59 }
  60 int dummy_register = AddContextCases();
  61 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
  62
  63 // ========================================================================= //
  64 // ------------------------ Testing Basic Output --------------------------- //
  65 // ========================================================================= //
  66
  67 void BM_basic(benchmark::State& state) {
  68   for (auto _ : state) {
  69   }
  70 }
  71 BENCHMARK(BM_basic);
  72
  73 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
  74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
  75                        {"\"family_index\": 0,$", MR_Next},
  76                        {"\"per_family_instance_index\": 0,$", MR_Next},
  77                        {"\"run_name\": \"BM_basic\",$", MR_Next},
  78                        {"\"run_type\": \"iteration\",$", MR_Next},
  79                        {"\"repetitions\": 1,$", MR_Next},
  80                        {"\"repetition_index\": 0,$", MR_Next},
  81                        {"\"threads\": 1,$", MR_Next},
  82                        {"\"iterations\": %int,$", MR_Next},
  83                        {"\"real_time\": %float,$", MR_Next},
  84                        {"\"cpu_time\": %float,$", MR_Next},
  85                        {"\"time_unit\": \"ns\"$", MR_Next},
  86                        {"}", MR_Next}});
  87 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
  88
  89 // ========================================================================= //
  90 // ------------------------ Testing Bytes per Second Output ---------------- //
  91 // ========================================================================= //
  92
  93 void BM_bytes_per_second(benchmark::State& state) {
  94   for (auto _ : state) {
  95     // This test requires a non-zero CPU time to avoid divide-by-zero
  96     benchmark::DoNotOptimize(state.iterations());
  97   }
  98   state.SetBytesProcessed(1);
  99 }
 100 BENCHMARK(BM_bytes_per_second);
 101
 102 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
 103                            "bytes_per_second=%float[kM]{0,1}/s$"}});
 104 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
 105                        {"\"family_index\": 1,$", MR_Next},
 106                        {"\"per_family_instance_index\": 0,$", MR_Next},
 107                        {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
 108                        {"\"run_type\": \"iteration\",$", MR_Next},
 109                        {"\"repetitions\": 1,$", MR_Next},
 110                        {"\"repetition_index\": 0,$", MR_Next},
 111                        {"\"threads\": 1,$", MR_Next},
 112                        {"\"iterations\": %int,$", MR_Next},
 113                        {"\"real_time\": %float,$", MR_Next},
 114                        {"\"cpu_time\": %float,$", MR_Next},
 115                        {"\"time_unit\": \"ns\",$", MR_Next},
 116                        {"\"bytes_per_second\": %float$", MR_Next},
 117                        {"}", MR_Next}});
 118 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
 119
 120 // ========================================================================= //
 121 // ------------------------ Testing Items per Second Output ---------------- //
 122 // ========================================================================= //
 123
 124 void BM_items_per_second(benchmark::State& state) {
 125   for (auto _ : state) {
 126     // This test requires a non-zero CPU time to avoid divide-by-zero
 127     benchmark::DoNotOptimize(state.iterations());
 128   }
 129   state.SetItemsProcessed(1);
 130 }
 131 BENCHMARK(BM_items_per_second);
 132
 133 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
 134                            "items_per_second=%float[kM]{0,1}/s$"}});
 135 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
 136                        {"\"family_index\": 2,$", MR_Next},
 137                        {"\"per_family_instance_index\": 0,$", MR_Next},
 138                        {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
 139                        {"\"run_type\": \"iteration\",$", MR_Next},
 140                        {"\"repetitions\": 1,$", MR_Next},
 141                        {"\"repetition_index\": 0,$", MR_Next},
 142                        {"\"threads\": 1,$", MR_Next},
 143                        {"\"iterations\": %int,$", MR_Next},
 144                        {"\"real_time\": %float,$", MR_Next},
 145                        {"\"cpu_time\": %float,$", MR_Next},
 146                        {"\"time_unit\": \"ns\",$", MR_Next},
 147                        {"\"items_per_second\": %float$", MR_Next},
 148                        {"}", MR_Next}});
 149 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
 150
 151 // ========================================================================= //
 152 // ------------------------ Testing Label Output --------------------------- //
 153 // ========================================================================= //
 154
 155 void BM_label(benchmark::State& state) {
 156   for (auto _ : state) {
 157   }
 158   state.SetLabel("some label");
 159 }
 160 BENCHMARK(BM_label);
 161
 162 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
 163 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
 164                        {"\"family_index\": 3,$", MR_Next},
 165                        {"\"per_family_instance_index\": 0,$", MR_Next},
 166                        {"\"run_name\": \"BM_label\",$", MR_Next},
 167                        {"\"run_type\": \"iteration\",$", MR_Next},
 168                        {"\"repetitions\": 1,$", MR_Next},
 169                        {"\"repetition_index\": 0,$", MR_Next},
 170                        {"\"threads\": 1,$", MR_Next},
 171                        {"\"iterations\": %int,$", MR_Next},
 172                        {"\"real_time\": %float,$", MR_Next},
 173                        {"\"cpu_time\": %float,$", MR_Next},
 174                        {"\"time_unit\": \"ns\",$", MR_Next},
 175                        {"\"label\": \"some label\"$", MR_Next},
 176                        {"}", MR_Next}});
 177 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
 178                        "label\"%csv_label_report_end$"}});
 179
 180 // ========================================================================= //
 181 // ------------------------ Testing Time Label Output ---------------------- //
 182 // ========================================================================= //
 183
 184 void BM_time_label_nanosecond(benchmark::State& state) {
 185   for (auto _ : state) {
 186   }
 187 }
 188 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
 189
 190 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
 191 ADD_CASES(TC_JSONOut,
 192           {{"\"name\": \"BM_time_label_nanosecond\",$"},
 193            {"\"family_index\": 4,$", MR_Next},
 194            {"\"per_family_instance_index\": 0,$", MR_Next},
 195            {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
 196            {"\"run_type\": \"iteration\",$", MR_Next},
 197            {"\"repetitions\": 1,$", MR_Next},
 198            {"\"repetition_index\": 0,$", MR_Next},
 199            {"\"threads\": 1,$", MR_Next},
 200            {"\"iterations\": %int,$", MR_Next},
 201            {"\"real_time\": %float,$", MR_Next},
 202            {"\"cpu_time\": %float,$", MR_Next},
 203            {"\"time_unit\": \"ns\"$", MR_Next},
 204            {"}", MR_Next}});
 205 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
 206
 207 void BM_time_label_microsecond(benchmark::State& state) {
 208   for (auto _ : state) {
 209   }
 210 }
 211 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
 212
 213 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
 214 ADD_CASES(TC_JSONOut,
 215           {{"\"name\": \"BM_time_label_microsecond\",$"},
 216            {"\"family_index\": 5,$", MR_Next},
 217            {"\"per_family_instance_index\": 0,$", MR_Next},
 218            {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
 219            {"\"run_type\": \"iteration\",$", MR_Next},
 220            {"\"repetitions\": 1,$", MR_Next},
 221            {"\"repetition_index\": 0,$", MR_Next},
 222            {"\"threads\": 1,$", MR_Next},
 223            {"\"iterations\": %int,$", MR_Next},
 224            {"\"real_time\": %float,$", MR_Next},
 225            {"\"cpu_time\": %float,$", MR_Next},
 226            {"\"time_unit\": \"us\"$", MR_Next},
 227            {"}", MR_Next}});
 228 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
 229
 230 void BM_time_label_millisecond(benchmark::State& state) {
 231   for (auto _ : state) {
 232   }
 233 }
 234 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
 235
 236 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
 237 ADD_CASES(TC_JSONOut,
 238           {{"\"name\": \"BM_time_label_millisecond\",$"},
 239            {"\"family_index\": 6,$", MR_Next},
 240            {"\"per_family_instance_index\": 0,$", MR_Next},
 241            {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
 242            {"\"run_type\": \"iteration\",$", MR_Next},
 243            {"\"repetitions\": 1,$", MR_Next},
 244            {"\"repetition_index\": 0,$", MR_Next},
 245            {"\"threads\": 1,$", MR_Next},
 246            {"\"iterations\": %int,$", MR_Next},
 247            {"\"real_time\": %float,$", MR_Next},
 248            {"\"cpu_time\": %float,$", MR_Next},
 249            {"\"time_unit\": \"ms\"$", MR_Next},
 250            {"}", MR_Next}});
 251 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
 252
 253 void BM_time_label_second(benchmark::State& state) {
 254   for (auto _ : state) {
 255   }
 256 }
 257 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
 258
 259 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
 260 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
 261                        {"\"family_index\": 7,$", MR_Next},
 262                        {"\"per_family_instance_index\": 0,$", MR_Next},
 263                        {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
 264                        {"\"run_type\": \"iteration\",$", MR_Next},
 265                        {"\"repetitions\": 1,$", MR_Next},
 266                        {"\"repetition_index\": 0,$", MR_Next},
 267                        {"\"threads\": 1,$", MR_Next},
 268                        {"\"iterations\": %int,$", MR_Next},
 269                        {"\"real_time\": %float,$", MR_Next},
 270                        {"\"cpu_time\": %float,$", MR_Next},
 271                        {"\"time_unit\": \"s\"$", MR_Next},
 272                        {"}", MR_Next}});
 273 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
 274
 275 // ========================================================================= //
 276 // ------------------------ Testing Error Output --------------------------- //
 277 // ========================================================================= //
 278
 279 void BM_error(benchmark::State& state) {
 280   state.SkipWithError("message");
 281   for (auto _ : state) {
 282   }
 283 }
 284 BENCHMARK(BM_error);
 285 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
 286 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
 287                        {"\"family_index\": 8,$", MR_Next},
 288                        {"\"per_family_instance_index\": 0,$", MR_Next},
 289                        {"\"run_name\": \"BM_error\",$", MR_Next},
 290                        {"\"run_type\": \"iteration\",$", MR_Next},
 291                        {"\"repetitions\": 1,$", MR_Next},
 292                        {"\"repetition_index\": 0,$", MR_Next},
 293                        {"\"threads\": 1,$", MR_Next},
 294                        {"\"error_occurred\": true,$", MR_Next},
 295                        {"\"error_message\": \"message\",$", MR_Next}});
 296
 297 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
 298
 299 // ========================================================================= //
 300 // ------------------------ Testing No Arg Name Output -----------------------
 301 // //
 302 // ========================================================================= //
 303
 304 void BM_no_arg_name(benchmark::State& state) {
 305   for (auto _ : state) {
 306   }
 307 }
 308 BENCHMARK(BM_no_arg_name)->Arg(3);
 309 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
 310 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
 311                        {"\"family_index\": 9,$", MR_Next},
 312                        {"\"per_family_instance_index\": 0,$", MR_Next},
 313                        {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
 314                        {"\"run_type\": \"iteration\",$", MR_Next},
 315                        {"\"repetitions\": 1,$", MR_Next},
 316                        {"\"repetition_index\": 0,$", MR_Next},
 317                        {"\"threads\": 1,$", MR_Next}});
 318 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
 319
 320 // ========================================================================= //
 321 // ------------------------ Testing Arg Name Output ----------------------- //
 322 // ========================================================================= //
 323
 324 void BM_arg_name(benchmark::State& state) {
 325   for (auto _ : state) {
 326   }
 327 }
 328 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
 329 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
 330 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
 331                        {"\"family_index\": 10,$", MR_Next},
 332                        {"\"per_family_instance_index\": 0,$", MR_Next},
 333                        {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
 334                        {"\"run_type\": \"iteration\",$", MR_Next},
 335                        {"\"repetitions\": 1,$", MR_Next},
 336                        {"\"repetition_index\": 0,$", MR_Next},
 337                        {"\"threads\": 1,$", MR_Next}});
 338 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
 339
 340 // ========================================================================= //
 341 // ------------------------ Testing Arg Names Output ----------------------- //
 342 // ========================================================================= //
 343
 344 void BM_arg_names(benchmark::State& state) {
 345   for (auto _ : state) {
 346   }
 347 }
 348 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
 349 ADD_CASES(TC_ConsoleOut,
 350           {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
 351 ADD_CASES(TC_JSONOut,
 352           {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
 353            {"\"family_index\": 11,$", MR_Next},
 354            {"\"per_family_instance_index\": 0,$", MR_Next},
 355            {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
 356            {"\"run_type\": \"iteration\",$", MR_Next},
 357            {"\"repetitions\": 1,$", MR_Next},
 358            {"\"repetition_index\": 0,$", MR_Next},
 359            {"\"threads\": 1,$", MR_Next}});
 360 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
 361
 362 // ========================================================================= //
 363 // ------------------------ Testing Name Output ---------------------------- //
 364 // ========================================================================= //
 365
 366 void BM_name(benchmark::State& state) {
 367   for (auto _ : state) {
 368   }
 369 }
 370 BENCHMARK(BM_name)->Name("BM_custom_name");
 371
 372 ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
 373 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
 374                        {"\"family_index\": 12,$", MR_Next},
 375                        {"\"per_family_instance_index\": 0,$", MR_Next},
 376                        {"\"run_name\": \"BM_custom_name\",$", MR_Next},
 377                        {"\"run_type\": \"iteration\",$", MR_Next},
 378                        {"\"repetitions\": 1,$", MR_Next},
 379                        {"\"repetition_index\": 0,$", MR_Next},
 380                        {"\"threads\": 1,$", MR_Next},
 381                        {"\"iterations\": %int,$", MR_Next},
 382                        {"\"real_time\": %float,$", MR_Next},
 383                        {"\"cpu_time\": %float,$", MR_Next},
 384                        {"\"time_unit\": \"ns\"$", MR_Next},
 385                        {"}", MR_Next}});
 386 ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
 387
 388 // ========================================================================= //
 389 // ------------------------ Testing Big Args Output ------------------------ //
 390 // ========================================================================= //
 391
 392 void BM_BigArgs(benchmark::State& state) {
 393   for (auto _ : state) {
 394   }
 395 }
 396 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
 397 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
 398                           {"^BM_BigArgs/2147483648 %console_report$"}});
 399
 400 // ========================================================================= //
 401 // ----------------------- Testing Complexity Output ----------------------- //
 402 // ========================================================================= //
 403
 404 void BM_Complexity_O1(benchmark::State& state) {
 405   for (auto _ : state) {
 406     // This test requires a non-zero CPU time to avoid divide-by-zero
 407     benchmark::DoNotOptimize(state.iterations());
 408   }
 409   state.SetComplexityN(state.range(0));
 410 }
 411 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
 412 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
 413                    {"%RMS", "[ ]*[0-9]+ %"}});
 414 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
 415                           {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
 416
 417 // ========================================================================= //
 418 // ----------------------- Testing Aggregate Output ------------------------ //
 419 // ========================================================================= //
 420
 421 // Test that non-aggregate data is printed by default
 422 void BM_Repeat(benchmark::State& state) {
 423   for (auto _ : state) {
 424   }
 425 }
 426 // need two repetitions min to be able to output any aggregate output
 427 BENCHMARK(BM_Repeat)->Repetitions(2);
 428 ADD_CASES(TC_ConsoleOut,
 429           {{"^BM_Repeat/repeats:2 %console_report$"},
 430            {"^BM_Repeat/repeats:2 %console_report$"},
 431            {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
 432            {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
 433            {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
 434 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
 435                        {"\"family_index\": 15,$", MR_Next},
 436                        {"\"per_family_instance_index\": 0,$", MR_Next},
 437                        {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
 438                        {"\"run_type\": \"iteration\",$", MR_Next},
 439                        {"\"repetitions\": 2,$", MR_Next},
 440                        {"\"repetition_index\": 0,$", MR_Next},
 441                        {"\"threads\": 1,$", MR_Next},
 442                        {"\"name\": \"BM_Repeat/repeats:2\",$"},
 443                        {"\"family_index\": 15,$", MR_Next},
 444                        {"\"per_family_instance_index\": 0,$", MR_Next},
 445                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 446                        {"\"run_type\": \"iteration\",$", MR_Next},
 447                        {"\"repetitions\": 2,$", MR_Next},
 448                        {"\"repetition_index\": 1,$", MR_Next},
 449                        {"\"threads\": 1,$", MR_Next},
 450                        {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
 451                        {"\"family_index\": 15,$", MR_Next},
 452                        {"\"per_family_instance_index\": 0,$", MR_Next},
 453                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 454                        {"\"run_type\": \"aggregate\",$", MR_Next},
 455                        {"\"repetitions\": 2,$", MR_Next},
 456                        {"\"threads\": 1,$", MR_Next},
 457                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 458                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 459                        {"\"iterations\": 2,$", MR_Next},
 460                        {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
 461                        {"\"family_index\": 15,$", MR_Next},
 462                        {"\"per_family_instance_index\": 0,$", MR_Next},
 463                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 464                        {"\"run_type\": \"aggregate\",$", MR_Next},
 465                        {"\"repetitions\": 2,$", MR_Next},
 466                        {"\"threads\": 1,$", MR_Next},
 467                        {"\"aggregate_name\": \"median\",$", MR_Next},
 468                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 469                        {"\"iterations\": 2,$", MR_Next},
 470                        {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
 471                        {"\"family_index\": 15,$", MR_Next},
 472                        {"\"per_family_instance_index\": 0,$", MR_Next},
 473                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
 474                        {"\"run_type\": \"aggregate\",$", MR_Next},
 475                        {"\"repetitions\": 2,$", MR_Next},
 476                        {"\"threads\": 1,$", MR_Next},
 477                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 478                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 479                        {"\"iterations\": 2,$", MR_Next}});
 480 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
 481                       {"^\"BM_Repeat/repeats:2\",%csv_report$"},
 482                       {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
 483                       {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
 484                       {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
 485 // but for two repetitions, mean and median is the same, so let's repeat..
 486 BENCHMARK(BM_Repeat)->Repetitions(3);
 487 ADD_CASES(TC_ConsoleOut,
 488           {{"^BM_Repeat/repeats:3 %console_report$"},
 489            {"^BM_Repeat/repeats:3 %console_report$"},
 490            {"^BM_Repeat/repeats:3 %console_report$"},
 491            {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
 492            {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
 493            {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
 494 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
 495                        {"\"family_index\": 16,$", MR_Next},
 496                        {"\"per_family_instance_index\": 0,$", MR_Next},
 497                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 498                        {"\"run_type\": \"iteration\",$", MR_Next},
 499                        {"\"repetitions\": 3,$", MR_Next},
 500                        {"\"repetition_index\": 0,$", MR_Next},
 501                        {"\"threads\": 1,$", MR_Next},
 502                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 503                        {"\"family_index\": 16,$", MR_Next},
 504                        {"\"per_family_instance_index\": 0,$", MR_Next},
 505                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 506                        {"\"run_type\": \"iteration\",$", MR_Next},
 507                        {"\"repetitions\": 3,$", MR_Next},
 508                        {"\"repetition_index\": 1,$", MR_Next},
 509                        {"\"threads\": 1,$", MR_Next},
 510                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
 511                        {"\"family_index\": 16,$", MR_Next},
 512                        {"\"per_family_instance_index\": 0,$", MR_Next},
 513                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 514                        {"\"run_type\": \"iteration\",$", MR_Next},
 515                        {"\"repetitions\": 3,$", MR_Next},
 516                        {"\"repetition_index\": 2,$", MR_Next},
 517                        {"\"threads\": 1,$", MR_Next},
 518                        {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
 519                        {"\"family_index\": 16,$", MR_Next},
 520                        {"\"per_family_instance_index\": 0,$", MR_Next},
 521                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 522                        {"\"run_type\": \"aggregate\",$", MR_Next},
 523                        {"\"repetitions\": 3,$", MR_Next},
 524                        {"\"threads\": 1,$", MR_Next},
 525                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 526                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 527                        {"\"iterations\": 3,$", MR_Next},
 528                        {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
 529                        {"\"family_index\": 16,$", MR_Next},
 530                        {"\"per_family_instance_index\": 0,$", MR_Next},
 531                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 532                        {"\"run_type\": \"aggregate\",$", MR_Next},
 533                        {"\"repetitions\": 3,$", MR_Next},
 534                        {"\"threads\": 1,$", MR_Next},
 535                        {"\"aggregate_name\": \"median\",$", MR_Next},
 536                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 537                        {"\"iterations\": 3,$", MR_Next},
 538                        {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
 539                        {"\"family_index\": 16,$", MR_Next},
 540                        {"\"per_family_instance_index\": 0,$", MR_Next},
 541                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
 542                        {"\"run_type\": \"aggregate\",$", MR_Next},
 543                        {"\"repetitions\": 3,$", MR_Next},
 544                        {"\"threads\": 1,$", MR_Next},
 545                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 546                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 547                        {"\"iterations\": 3,$", MR_Next}});
 548 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
 549                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 550                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
 551                       {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
 552                       {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
 553                       {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
 554 // median differs between even/odd number of repetitions, so just to be sure
 555 BENCHMARK(BM_Repeat)->Repetitions(4);
 556 ADD_CASES(TC_ConsoleOut,
 557           {{"^BM_Repeat/repeats:4 %console_report$"},
 558            {"^BM_Repeat/repeats:4 %console_report$"},
 559            {"^BM_Repeat/repeats:4 %console_report$"},
 560            {"^BM_Repeat/repeats:4 %console_report$"},
 561            {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
 562            {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
 563            {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
 564 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
 565                        {"\"family_index\": 17,$", MR_Next},
 566                        {"\"per_family_instance_index\": 0,$", MR_Next},
 567                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 568                        {"\"run_type\": \"iteration\",$", MR_Next},
 569                        {"\"repetitions\": 4,$", MR_Next},
 570                        {"\"repetition_index\": 0,$", MR_Next},
 571                        {"\"threads\": 1,$", MR_Next},
 572                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 573                        {"\"family_index\": 17,$", MR_Next},
 574                        {"\"per_family_instance_index\": 0,$", MR_Next},
 575                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 576                        {"\"run_type\": \"iteration\",$", MR_Next},
 577                        {"\"repetitions\": 4,$", MR_Next},
 578                        {"\"repetition_index\": 1,$", MR_Next},
 579                        {"\"threads\": 1,$", MR_Next},
 580                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 581                        {"\"family_index\": 17,$", MR_Next},
 582                        {"\"per_family_instance_index\": 0,$", MR_Next},
 583                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 584                        {"\"run_type\": \"iteration\",$", MR_Next},
 585                        {"\"repetitions\": 4,$", MR_Next},
 586                        {"\"repetition_index\": 2,$", MR_Next},
 587                        {"\"threads\": 1,$", MR_Next},
 588                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
 589                        {"\"family_index\": 17,$", MR_Next},
 590                        {"\"per_family_instance_index\": 0,$", MR_Next},
 591                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 592                        {"\"run_type\": \"iteration\",$", MR_Next},
 593                        {"\"repetitions\": 4,$", MR_Next},
 594                        {"\"repetition_index\": 3,$", MR_Next},
 595                        {"\"threads\": 1,$", MR_Next},
 596                        {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
 597                        {"\"family_index\": 17,$", MR_Next},
 598                        {"\"per_family_instance_index\": 0,$", MR_Next},
 599                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 600                        {"\"run_type\": \"aggregate\",$", MR_Next},
 601                        {"\"repetitions\": 4,$", MR_Next},
 602                        {"\"threads\": 1,$", MR_Next},
 603                        {"\"aggregate_name\": \"mean\",$", MR_Next},
 604                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 605                        {"\"iterations\": 4,$", MR_Next},
 606                        {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
 607                        {"\"family_index\": 17,$", MR_Next},
 608                        {"\"per_family_instance_index\": 0,$", MR_Next},
 609                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 610                        {"\"run_type\": \"aggregate\",$", MR_Next},
 611                        {"\"repetitions\": 4,$", MR_Next},
 612                        {"\"threads\": 1,$", MR_Next},
 613                        {"\"aggregate_name\": \"median\",$", MR_Next},
 614                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 615                        {"\"iterations\": 4,$", MR_Next},
 616                        {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
 617                        {"\"family_index\": 17,$", MR_Next},
 618                        {"\"per_family_instance_index\": 0,$", MR_Next},
 619                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
 620                        {"\"run_type\": \"aggregate\",$", MR_Next},
 621                        {"\"repetitions\": 4,$", MR_Next},
 622                        {"\"threads\": 1,$", MR_Next},
 623                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
 624                        {"\"aggregate_unit\": \"time\",$", MR_Next},
 625                        {"\"iterations\": 4,$", MR_Next}});
 626 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
 627                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 628                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 629                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
 630                       {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
 631                       {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
 632                       {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
 633
 634 // Test that a non-repeated test still prints non-aggregate results even when
 635 // only-aggregate reports have been requested
 636 void BM_RepeatOnce(benchmark::State& state) {
 637   for (auto _ : state) {
 638   }
 639 }
 640 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
 641 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
 642 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
 643                        {"\"family_index\": 18,$", MR_Next},
 644                        {"\"per_family_instance_index\": 0,$", MR_Next},
 645                        {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
 646                        {"\"run_type\": \"iteration\",$", MR_Next},
 647                        {"\"repetitions\": 1,$", MR_Next},
 648                        {"\"repetition_index\": 0,$", MR_Next},
 649                        {"\"threads\": 1,$", MR_Next}});
 650 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
 651
 652 // Test that non-aggregate data is not reported
 653 void BM_SummaryRepeat(benchmark::State& state) {
 654   for (auto _ : state) {
 655   }
 656 }
 657 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
 658 ADD_CASES(
 659     TC_ConsoleOut,
 660     {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 661      {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
 662      {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
 663      {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
 664 ADD_CASES(TC_JSONOut,
 665           {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 666            {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
 667            {"\"family_index\": 19,$", MR_Next},
 668            {"\"per_family_instance_index\": 0,$", MR_Next},
 669            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 670            {"\"run_type\": \"aggregate\",$", MR_Next},
 671            {"\"repetitions\": 3,$", MR_Next},
 672            {"\"threads\": 1,$", MR_Next},
 673            {"\"aggregate_name\": \"mean\",$", MR_Next},
 674            {"\"aggregate_unit\": \"time\",$", MR_Next},
 675            {"\"iterations\": 3,$", MR_Next},
 676            {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
 677            {"\"family_index\": 19,$", MR_Next},
 678            {"\"per_family_instance_index\": 0,$", MR_Next},
 679            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 680            {"\"run_type\": \"aggregate\",$", MR_Next},
 681            {"\"repetitions\": 3,$", MR_Next},
 682            {"\"threads\": 1,$", MR_Next},
 683            {"\"aggregate_name\": \"median\",$", MR_Next},
 684            {"\"aggregate_unit\": \"time\",$", MR_Next},
 685            {"\"iterations\": 3,$", MR_Next},
 686            {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
 687            {"\"family_index\": 19,$", MR_Next},
 688            {"\"per_family_instance_index\": 0,$", MR_Next},
 689            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
 690            {"\"run_type\": \"aggregate\",$", MR_Next},
 691            {"\"repetitions\": 3,$", MR_Next},
 692            {"\"threads\": 1,$", MR_Next},
 693            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 694            {"\"aggregate_unit\": \"time\",$", MR_Next},
 695            {"\"iterations\": 3,$", MR_Next}});
 696 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
 697                       {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
 698                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
 699                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
 700
 701 // Test that non-aggregate data is not displayed.
 702 // NOTE: this test is kinda bad. we are only testing the display output.
 703 //       But we don't check that the file output still contains everything...
 704 void BM_SummaryDisplay(benchmark::State& state) {
 705   for (auto _ : state) {
 706   }
 707 }
 708 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
 709 ADD_CASES(
 710     TC_ConsoleOut,
 711     {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 712      {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
 713      {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
 714      {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
 715 ADD_CASES(TC_JSONOut,
 716           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 717            {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
 718            {"\"family_index\": 20,$", MR_Next},
 719            {"\"per_family_instance_index\": 0,$", MR_Next},
 720            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 721            {"\"run_type\": \"aggregate\",$", MR_Next},
 722            {"\"repetitions\": 2,$", MR_Next},
 723            {"\"threads\": 1,$", MR_Next},
 724            {"\"aggregate_name\": \"mean\",$", MR_Next},
 725            {"\"aggregate_unit\": \"time\",$", MR_Next},
 726            {"\"iterations\": 2,$", MR_Next},
 727            {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
 728            {"\"family_index\": 20,$", MR_Next},
 729            {"\"per_family_instance_index\": 0,$", MR_Next},
 730            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 731            {"\"run_type\": \"aggregate\",$", MR_Next},
 732            {"\"repetitions\": 2,$", MR_Next},
 733            {"\"threads\": 1,$", MR_Next},
 734            {"\"aggregate_name\": \"median\",$", MR_Next},
 735            {"\"aggregate_unit\": \"time\",$", MR_Next},
 736            {"\"iterations\": 2,$", MR_Next},
 737            {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
 738            {"\"family_index\": 20,$", MR_Next},
 739            {"\"per_family_instance_index\": 0,$", MR_Next},
 740            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
 741            {"\"run_type\": \"aggregate\",$", MR_Next},
 742            {"\"repetitions\": 2,$", MR_Next},
 743            {"\"threads\": 1,$", MR_Next},
 744            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 745            {"\"aggregate_unit\": \"time\",$", MR_Next},
 746            {"\"iterations\": 2,$", MR_Next}});
 747 ADD_CASES(TC_CSVOut,
 748           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
 749            {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
 750            {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
 751            {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
 752
 753 // Test repeats with custom time unit.
 754 void BM_RepeatTimeUnit(benchmark::State& state) {
 755   for (auto _ : state) {
 756   }
 757 }
 758 BENCHMARK(BM_RepeatTimeUnit)
 759     ->Repetitions(3)
 760     ->ReportAggregatesOnly()
 761     ->Unit(benchmark::kMicrosecond);
 762 ADD_CASES(
 763     TC_ConsoleOut,
 764     {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 765      {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
 766      {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
 767       "]*3$"},
 768      {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
 769       "]*3$"}});
 770 ADD_CASES(TC_JSONOut,
 771           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 772            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
 773            {"\"family_index\": 21,$", MR_Next},
 774            {"\"per_family_instance_index\": 0,$", MR_Next},
 775            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 776            {"\"run_type\": \"aggregate\",$", MR_Next},
 777            {"\"repetitions\": 3,$", MR_Next},
 778            {"\"threads\": 1,$", MR_Next},
 779            {"\"aggregate_name\": \"mean\",$", MR_Next},
 780            {"\"aggregate_unit\": \"time\",$", MR_Next},
 781            {"\"iterations\": 3,$", MR_Next},
 782            {"\"time_unit\": \"us\",?$"},
 783            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
 784            {"\"family_index\": 21,$", MR_Next},
 785            {"\"per_family_instance_index\": 0,$", MR_Next},
 786            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 787            {"\"run_type\": \"aggregate\",$", MR_Next},
 788            {"\"repetitions\": 3,$", MR_Next},
 789            {"\"threads\": 1,$", MR_Next},
 790            {"\"aggregate_name\": \"median\",$", MR_Next},
 791            {"\"aggregate_unit\": \"time\",$", MR_Next},
 792            {"\"iterations\": 3,$", MR_Next},
 793            {"\"time_unit\": \"us\",?$"},
 794            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
 795            {"\"family_index\": 21,$", MR_Next},
 796            {"\"per_family_instance_index\": 0,$", MR_Next},
 797            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
 798            {"\"run_type\": \"aggregate\",$", MR_Next},
 799            {"\"repetitions\": 3,$", MR_Next},
 800            {"\"threads\": 1,$", MR_Next},
 801            {"\"aggregate_name\": \"stddev\",$", MR_Next},
 802            {"\"aggregate_unit\": \"time\",$", MR_Next},
 803            {"\"iterations\": 3,$", MR_Next},
 804            {"\"time_unit\": \"us\",?$"}});
 805 ADD_CASES(TC_CSVOut,
 806           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
 807            {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
 808            {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
 809            {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
 810
 811 // ========================================================================= //
 812 // -------------------- Testing user-provided statistics ------------------- //
 813 // ========================================================================= //
 814
 815 const auto UserStatistics = [](const std::vector<double>& v) {
 816   return v.back();
 817 };
 818 void BM_UserStats(benchmark::State& state) {
 819   for (auto _ : state) {
 820     state.SetIterationTime(150 / 10e8);
 821   }
 822 }
 823 // clang-format off
 824 BENCHMARK(BM_UserStats)
 825   ->Repetitions(3)
 826   ->Iterations(5)
 827   ->UseManualTime()
 828   ->ComputeStatistics("", UserStatistics);
 829 // clang-format on
 830
 831 // check that user-provided stats is calculated, and is after the default-ones
 832 // empty string as name is intentional, it would sort before anything else
 833 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 834                            "]* 150 ns %time [ ]*5$"},
 835                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 836                            "]* 150 ns %time [ ]*5$"},
 837                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
 838                            "]* 150 ns %time [ ]*5$"},
 839                           {"^BM_UserStats/iterations:5/repeats:3/"
 840                            "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
 841                           {"^BM_UserStats/iterations:5/repeats:3/"
 842                            "manual_time_median [ ]* 150 ns %time [ ]*3$"},
 843                           {"^BM_UserStats/iterations:5/repeats:3/"
 844                            "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
 845                           {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
 846                            "[ ]* 150 ns %time [ ]*3$"}});
 847 ADD_CASES(
 848     TC_JSONOut,
 849     {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 850      {"\"family_index\": 22,$", MR_Next},
 851      {"\"per_family_instance_index\": 0,$", MR_Next},
 852      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 853       MR_Next},
 854      {"\"run_type\": \"iteration\",$", MR_Next},
 855      {"\"repetitions\": 3,$", MR_Next},
 856      {"\"repetition_index\": 0,$", MR_Next},
 857      {"\"threads\": 1,$", MR_Next},
 858      {"\"iterations\": 5,$", MR_Next},
 859      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 860      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 861      {"\"family_index\": 22,$", MR_Next},
 862      {"\"per_family_instance_index\": 0,$", MR_Next},
 863      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 864       MR_Next},
 865      {"\"run_type\": \"iteration\",$", MR_Next},
 866      {"\"repetitions\": 3,$", MR_Next},
 867      {"\"repetition_index\": 1,$", MR_Next},
 868      {"\"threads\": 1,$", MR_Next},
 869      {"\"iterations\": 5,$", MR_Next},
 870      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 871      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
 872      {"\"family_index\": 22,$", MR_Next},
 873      {"\"per_family_instance_index\": 0,$", MR_Next},
 874      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 875       MR_Next},
 876      {"\"run_type\": \"iteration\",$", MR_Next},
 877      {"\"repetitions\": 3,$", MR_Next},
 878      {"\"repetition_index\": 2,$", MR_Next},
 879      {"\"threads\": 1,$", MR_Next},
 880      {"\"iterations\": 5,$", MR_Next},
 881      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 882      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
 883      {"\"family_index\": 22,$", MR_Next},
 884      {"\"per_family_instance_index\": 0,$", MR_Next},
 885      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 886       MR_Next},
 887      {"\"run_type\": \"aggregate\",$", MR_Next},
 888      {"\"repetitions\": 3,$", MR_Next},
 889      {"\"threads\": 1,$", MR_Next},
 890      {"\"aggregate_name\": \"mean\",$", MR_Next},
 891      {"\"aggregate_unit\": \"time\",$", MR_Next},
 892      {"\"iterations\": 3,$", MR_Next},
 893      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 894      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
 895      {"\"family_index\": 22,$", MR_Next},
 896      {"\"per_family_instance_index\": 0,$", MR_Next},
 897      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 898       MR_Next},
 899      {"\"run_type\": \"aggregate\",$", MR_Next},
 900      {"\"repetitions\": 3,$", MR_Next},
 901      {"\"threads\": 1,$", MR_Next},
 902      {"\"aggregate_name\": \"median\",$", MR_Next},
 903      {"\"aggregate_unit\": \"time\",$", MR_Next},
 904      {"\"iterations\": 3,$", MR_Next},
 905      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 906      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
 907      {"\"family_index\": 22,$", MR_Next},
 908      {"\"per_family_instance_index\": 0,$", MR_Next},
 909      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 910       MR_Next},
 911      {"\"run_type\": \"aggregate\",$", MR_Next},
 912      {"\"repetitions\": 3,$", MR_Next},
 913      {"\"threads\": 1,$", MR_Next},
 914      {"\"aggregate_name\": \"stddev\",$", MR_Next},
 915      {"\"aggregate_unit\": \"time\",$", MR_Next},
 916      {"\"iterations\": 3,$", MR_Next},
 917      {"\"real_time\": %float,$", MR_Next},
 918      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
 919      {"\"family_index\": 22,$", MR_Next},
 920      {"\"per_family_instance_index\": 0,$", MR_Next},
 921      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
 922       MR_Next},
 923      {"\"run_type\": \"aggregate\",$", MR_Next},
 924      {"\"repetitions\": 3,$", MR_Next},
 925      {"\"threads\": 1,$", MR_Next},
 926      {"\"aggregate_name\": \"\",$", MR_Next},
 927      {"\"aggregate_unit\": \"time\",$", MR_Next},
 928      {"\"iterations\": 3,$", MR_Next},
 929      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
 930 ADD_CASES(
 931     TC_CSVOut,
 932     {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 933      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 934      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
 935      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
 936      {"^\"BM_UserStats/iterations:5/repeats:3/"
 937       "manual_time_median\",%csv_report$"},
 938      {"^\"BM_UserStats/iterations:5/repeats:3/"
 939       "manual_time_stddev\",%csv_report$"},
 940      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
 941
 942 // ========================================================================= //
 943 // ------------- Testing relative standard deviation statistics ------------ //
 944 // ========================================================================= //
 945
 946 const auto UserPercentStatistics = [](const std::vector<double>&) {
 947   return 1. / 100.;
 948 };
 949 void BM_UserPercentStats(benchmark::State& state) {
 950   for (auto _ : state) {
 951     state.SetIterationTime(150 / 10e8);
 952   }
 953 }
 954 // clang-format off
 955 BENCHMARK(BM_UserPercentStats)
 956   ->Repetitions(3)
 957   ->Iterations(5)
 958   ->UseManualTime()
 959   ->Unit(benchmark::TimeUnit::kNanosecond)
 960   ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
 961 // clang-format on
 962
 963 // check that UserPercent-provided stats is calculated, and is after the
 964 // default-ones empty string as name is intentional, it would sort before
 965 // anything else
 966 ADD_CASES(TC_ConsoleOut,
 967           {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 968             "]* 150 ns %time [ ]*5$"},
 969            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 970             "]* 150 ns %time [ ]*5$"},
 971            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
 972             "]* 150 ns %time [ ]*5$"},
 973            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 974             "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
 975            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 976             "manual_time_median [ ]* 150 ns %time [ ]*3$"},
 977            {"^BM_UserPercentStats/iterations:5/repeats:3/"
 978             "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
 979            {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
 980             "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
 981 ADD_CASES(
 982     TC_JSONOut,
 983     {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
 984      {"\"family_index\": 23,$", MR_Next},
 985      {"\"per_family_instance_index\": 0,$", MR_Next},
 986      {"\"run_name\": "
 987       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
 988       MR_Next},
 989      {"\"run_type\": \"iteration\",$", MR_Next},
 990      {"\"repetitions\": 3,$", MR_Next},
 991      {"\"repetition_index\": 0,$", MR_Next},
 992      {"\"threads\": 1,$", MR_Next},
 993      {"\"iterations\": 5,$", MR_Next},
 994      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
 995      {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
 996      {"\"family_index\": 23,$", MR_Next},
 997      {"\"per_family_instance_index\": 0,$", MR_Next},
 998      {"\"run_name\": "
 999       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1000       MR_Next},
1001      {"\"run_type\": \"iteration\",$", MR_Next},
1002      {"\"repetitions\": 3,$", MR_Next},
1003      {"\"repetition_index\": 1,$", MR_Next},
1004      {"\"threads\": 1,$", MR_Next},
1005      {"\"iterations\": 5,$", MR_Next},
1006      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1007      {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1008      {"\"family_index\": 23,$", MR_Next},
1009      {"\"per_family_instance_index\": 0,$", MR_Next},
1010      {"\"run_name\": "
1011       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1012       MR_Next},
1013      {"\"run_type\": \"iteration\",$", MR_Next},
1014      {"\"repetitions\": 3,$", MR_Next},
1015      {"\"repetition_index\": 2,$", MR_Next},
1016      {"\"threads\": 1,$", MR_Next},
1017      {"\"iterations\": 5,$", MR_Next},
1018      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1019      {"\"name\": "
1020       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1021      {"\"family_index\": 23,$", MR_Next},
1022      {"\"per_family_instance_index\": 0,$", MR_Next},
1023      {"\"run_name\": "
1024       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1025       MR_Next},
1026      {"\"run_type\": \"aggregate\",$", MR_Next},
1027      {"\"repetitions\": 3,$", MR_Next},
1028      {"\"threads\": 1,$", MR_Next},
1029      {"\"aggregate_name\": \"mean\",$", MR_Next},
1030      {"\"aggregate_unit\": \"time\",$", MR_Next},
1031      {"\"iterations\": 3,$", MR_Next},
1032      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1033      {"\"name\": "
1034       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1035      {"\"family_index\": 23,$", MR_Next},
1036      {"\"per_family_instance_index\": 0,$", MR_Next},
1037      {"\"run_name\": "
1038       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1039       MR_Next},
1040      {"\"run_type\": \"aggregate\",$", MR_Next},
1041      {"\"repetitions\": 3,$", MR_Next},
1042      {"\"threads\": 1,$", MR_Next},
1043      {"\"aggregate_name\": \"median\",$", MR_Next},
1044      {"\"aggregate_unit\": \"time\",$", MR_Next},
1045      {"\"iterations\": 3,$", MR_Next},
1046      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1047      {"\"name\": "
1048       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1049      {"\"family_index\": 23,$", MR_Next},
1050      {"\"per_family_instance_index\": 0,$", MR_Next},
1051      {"\"run_name\": "
1052       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1053       MR_Next},
1054      {"\"run_type\": \"aggregate\",$", MR_Next},
1055      {"\"repetitions\": 3,$", MR_Next},
1056      {"\"threads\": 1,$", MR_Next},
1057      {"\"aggregate_name\": \"stddev\",$", MR_Next},
1058      {"\"aggregate_unit\": \"time\",$", MR_Next},
1059      {"\"iterations\": 3,$", MR_Next},
1060      {"\"real_time\": %float,$", MR_Next},
1061      {"\"name\": "
1062       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1063      {"\"family_index\": 23,$", MR_Next},
1064      {"\"per_family_instance_index\": 0,$", MR_Next},
1065      {"\"run_name\": "
1066       "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1067       MR_Next},
1068      {"\"run_type\": \"aggregate\",$", MR_Next},
1069      {"\"repetitions\": 3,$", MR_Next},
1070      {"\"threads\": 1,$", MR_Next},
1071      {"\"aggregate_name\": \"\",$", MR_Next},
1072      {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1073      {"\"iterations\": 3,$", MR_Next},
1074      {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1075 ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1076                        "manual_time\",%csv_report$"},
1077                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1078                        "manual_time\",%csv_report$"},
1079                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1080                        "manual_time\",%csv_report$"},
1081                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1082                        "manual_time_mean\",%csv_report$"},
1083                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1084                        "manual_time_median\",%csv_report$"},
1085                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1086                        "manual_time_stddev\",%csv_report$"},
1087                       {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1088                        "manual_time_\",%csv_report$"}});
1089
1090 // ========================================================================= //
1091 // ------------------------- Testing StrEscape JSON ------------------------ //
1092 // ========================================================================= //
1093 #if 0  // enable when csv testing code correctly handles multi-line fields
1094 void BM_JSON_Format(benchmark::State& state) {
1095   state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1096   for (auto _ : state) {
1097   }
1098 }
1099 BENCHMARK(BM_JSON_Format);
1100 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1101                                               {"\"family_index\": 23,$", MR_Next},
1102 {"\"per_family_instance_index\": 0,$", MR_Next},
1103                        {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1104                        {"\"run_type\": \"iteration\",$", MR_Next},
1105                        {"\"repetitions\": 1,$", MR_Next},
1106                        {"\"repetition_index\": 0,$", MR_Next},
1107                        {"\"threads\": 1,$", MR_Next},
1108                        {"\"error_occurred\": true,$", MR_Next},
1109                        {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1110 #endif
1111 // ========================================================================= //
1112 // -------------------------- Testing CsvEscape ---------------------------- //
1113 // ========================================================================= //
1114
1115 void BM_CSV_Format(benchmark::State& state) {
1116   state.SkipWithError("\"freedom\"");
1117   for (auto _ : state) {
1118   }
1119 }
1120 BENCHMARK(BM_CSV_Format);
1121 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1122
1123 // ========================================================================= //
1124 // --------------------------- TEST CASES END ------------------------------ //
1125 // ========================================================================= //
1126
1127 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }