utils/benchmark/src/benchmark_register.cc

   1 // Copyright 2015 Google Inc. All rights reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 #include "benchmark_register.h"
  16
  17 #ifndef BENCHMARK_OS_WINDOWS
  18 #ifndef BENCHMARK_OS_FUCHSIA
  19 #include <sys/resource.h>
  20 #endif
  21 #include <sys/time.h>
  22 #include <unistd.h>
  23 #endif
  24
  25 #include <algorithm>
  26 #include <atomic>
  27 #include <condition_variable>
  28 #include <cstdio>
  29 #include <cstdlib>
  30 #include <cstring>
  31 #include <fstream>
  32 #include <iostream>
  33 #include <memory>
  34 #include <sstream>
  35 #include <thread>
  36
  37 #include "benchmark/benchmark.h"
  38 #include "benchmark_api_internal.h"
  39 #include "check.h"
  40 #include "commandlineflags.h"
  41 #include "complexity.h"
  42 #include "internal_macros.h"
  43 #include "log.h"
  44 #include "mutex.h"
  45 #include "re.h"
  46 #include "statistics.h"
  47 #include "string_util.h"
  48 #include "timers.h"
  49
  50 namespace benchmark {
  51
  52 namespace {
  53 // For non-dense Range, intermediate values are powers of kRangeMultiplier.
  54 static const int kRangeMultiplier = 8;
  55 // The size of a benchmark family determines is the number of inputs to repeat
  56 // the benchmark on. If this is "large" then warn the user during configuration.
  57 static const size_t kMaxFamilySize = 100;
  58 }  // end namespace
  59
  60 namespace internal {
  61
  62 //=============================================================================//
  63 //                         BenchmarkFamilies
  64 //=============================================================================//
  65
  66 // Class for managing registered benchmarks.  Note that each registered
  67 // benchmark identifies a family of related benchmarks to run.
  68 class BenchmarkFamilies {
  69  public:
  70   static BenchmarkFamilies* GetInstance();
  71
  72   // Registers a benchmark family and returns the index assigned to it.
  73   size_t AddBenchmark(std::unique_ptr<Benchmark> family);
  74
  75   // Clear all registered benchmark families.
  76   void ClearBenchmarks();
  77
  78   // Extract the list of benchmark instances that match the specified
  79   // regular expression.
  80   bool FindBenchmarks(std::string re,
  81                       std::vector<Benchmark::Instance>* benchmarks,
  82                       std::ostream* Err);
  83
  84  private:
  85   BenchmarkFamilies() {}
  86
  87   std::vector<std::unique_ptr<Benchmark>> families_;
  88   Mutex mutex_;
  89 };
  90
  91 BenchmarkFamilies* BenchmarkFamilies::GetInstance() {
  92   static BenchmarkFamilies instance;
  93   return &instance;
  94 }
  95
  96 size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr<Benchmark> family) {
  97   MutexLock l(mutex_);
  98   size_t index = families_.size();
  99   families_.push_back(std::move(family));
 100   return index;
 101 }
 102
 103 void BenchmarkFamilies::ClearBenchmarks() {
 104   MutexLock l(mutex_);
 105   families_.clear();
 106   families_.shrink_to_fit();
 107 }
 108
 109 bool BenchmarkFamilies::FindBenchmarks(
 110     std::string spec, std::vector<Benchmark::Instance>* benchmarks,
 111     std::ostream* ErrStream) {
 112   CHECK(ErrStream);
 113   auto& Err = *ErrStream;
 114   // Make regular expression out of command-line flag
 115   std::string error_msg;
 116   Regex re;
 117   bool isNegativeFilter = false;
 118   if(spec[0] == '-') {
 119       spec.replace(0, 1, "");
 120       isNegativeFilter = true;
 121   }
 122   if (!re.Init(spec, &error_msg)) {
 123     Err << "Could not compile benchmark re: " << error_msg << std::endl;
 124     return false;
 125   }
 126
 127   // Special list of thread counts to use when none are specified
 128   const std::vector<int> one_thread = {1};
 129
 130   MutexLock l(mutex_);
 131   for (std::unique_ptr<Benchmark>& family : families_) {
 132     // Family was deleted or benchmark doesn't match
 133     if (!family) continue;
 134
 135     if (family->ArgsCnt() == -1) {
 136       family->Args({});
 137     }
 138     const std::vector<int>* thread_counts =
 139         (family->thread_counts_.empty()
 140              ? &one_thread
 141              : &static_cast<const std::vector<int>&>(family->thread_counts_));
 142     const size_t family_size = family->args_.size() * thread_counts->size();
 143     // The benchmark will be run at least 'family_size' different inputs.
 144     // If 'family_size' is very large warn the user.
 145     if (family_size > kMaxFamilySize) {
 146       Err << "The number of inputs is very large. " << family->name_
 147           << " will be repeated at least " << family_size << " times.\n";
 148     }
 149     // reserve in the special case the regex ".", since we know the final
 150     // family size.
 151     if (spec == ".") benchmarks->reserve(family_size);
 152
 153     for (auto const& args : family->args_) {
 154       for (int num_threads : *thread_counts) {
 155         Benchmark::Instance instance;
 156         instance.name = family->name_;
 157         instance.benchmark = family.get();
 158         instance.report_mode = family->report_mode_;
 159         instance.arg = args;
 160         instance.time_unit = family->time_unit_;
 161         instance.range_multiplier = family->range_multiplier_;
 162         instance.min_time = family->min_time_;
 163         instance.iterations = family->iterations_;
 164         instance.repetitions = family->repetitions_;
 165         instance.use_real_time = family->use_real_time_;
 166         instance.use_manual_time = family->use_manual_time_;
 167         instance.complexity = family->complexity_;
 168         instance.complexity_lambda = family->complexity_lambda_;
 169         instance.statistics = &family->statistics_;
 170         instance.threads = num_threads;
 171
 172         // Add arguments to instance name
 173         size_t arg_i = 0;
 174         for (auto const& arg : args) {
 175           instance.name += "/";
 176
 177           if (arg_i < family->arg_names_.size()) {
 178             const auto& arg_name = family->arg_names_[arg_i];
 179             if (!arg_name.empty()) {
 180               instance.name +=
 181                   StrFormat("%s:", family->arg_names_[arg_i].c_str());
 182             }
 183           }
 184
 185           instance.name += StrFormat("%d", arg);
 186           ++arg_i;
 187         }
 188
 189         if (!IsZero(family->min_time_))
 190           instance.name += StrFormat("/min_time:%0.3f", family->min_time_);
 191         if (family->iterations_ != 0)
 192           instance.name += StrFormat("/iterations:%d", family->iterations_);
 193         if (family->repetitions_ != 0)
 194           instance.name += StrFormat("/repeats:%d", family->repetitions_);
 195
 196         if (family->use_manual_time_) {
 197           instance.name += "/manual_time";
 198         } else if (family->use_real_time_) {
 199           instance.name += "/real_time";
 200         }
 201
 202         // Add the number of threads used to the name
 203         if (!family->thread_counts_.empty()) {
 204           instance.name += StrFormat("/threads:%d", instance.threads);
 205         }
 206
 207         if ((re.Match(instance.name) && !isNegativeFilter) ||
 208             (!re.Match(instance.name) && isNegativeFilter)) {
 209           instance.last_benchmark_instance = (&args == &family->args_.back());
 210           benchmarks->push_back(std::move(instance));
 211         }
 212       }
 213     }
 214   }
 215   return true;
 216 }
 217
 218 Benchmark* RegisterBenchmarkInternal(Benchmark* bench) {
 219   std::unique_ptr<Benchmark> bench_ptr(bench);
 220   BenchmarkFamilies* families = BenchmarkFamilies::GetInstance();
 221   families->AddBenchmark(std::move(bench_ptr));
 222   return bench;
 223 }
 224
 225 // FIXME: This function is a hack so that benchmark.cc can access
 226 // `BenchmarkFamilies`
 227 bool FindBenchmarksInternal(const std::string& re,
 228                             std::vector<Benchmark::Instance>* benchmarks,
 229                             std::ostream* Err) {
 230   return BenchmarkFamilies::GetInstance()->FindBenchmarks(re, benchmarks, Err);
 231 }
 232
 233 //=============================================================================//
 234 //                               Benchmark
 235 //=============================================================================//
 236
 237 Benchmark::Benchmark(const char* name)
 238     : name_(name),
 239       report_mode_(RM_Unspecified),
 240       time_unit_(kNanosecond),
 241       range_multiplier_(kRangeMultiplier),
 242       min_time_(0),
 243       iterations_(0),
 244       repetitions_(0),
 245       use_real_time_(false),
 246       use_manual_time_(false),
 247       complexity_(oNone),
 248       complexity_lambda_(nullptr) {
 249   ComputeStatistics("mean", StatisticsMean);
 250   ComputeStatistics("median", StatisticsMedian);
 251   ComputeStatistics("stddev", StatisticsStdDev);
 252 }
 253
 254 Benchmark::~Benchmark() {}
 255
 256 Benchmark* Benchmark::Arg(int64_t x) {
 257   CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
 258   args_.push_back({x});
 259   return this;
 260 }
 261
 262 Benchmark* Benchmark::Unit(TimeUnit unit) {
 263   time_unit_ = unit;
 264   return this;
 265 }
 266
 267 Benchmark* Benchmark::Range(int64_t start, int64_t limit) {
 268   CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
 269   std::vector<int64_t> arglist;
 270   AddRange(&arglist, start, limit, range_multiplier_);
 271
 272   for (int64_t i : arglist) {
 273     args_.push_back({i});
 274   }
 275   return this;
 276 }
 277
 278 Benchmark* Benchmark::Ranges(
 279     const std::vector<std::pair<int64_t, int64_t>>& ranges) {
 280   CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(ranges.size()));
 281   std::vector<std::vector<int64_t>> arglists(ranges.size());
 282   std::size_t total = 1;
 283   for (std::size_t i = 0; i < ranges.size(); i++) {
 284     AddRange(&arglists[i], ranges[i].first, ranges[i].second,
 285              range_multiplier_);
 286     total *= arglists[i].size();
 287   }
 288
 289   std::vector<std::size_t> ctr(arglists.size(), 0);
 290
 291   for (std::size_t i = 0; i < total; i++) {
 292     std::vector<int64_t> tmp;
 293     tmp.reserve(arglists.size());
 294
 295     for (std::size_t j = 0; j < arglists.size(); j++) {
 296       tmp.push_back(arglists[j].at(ctr[j]));
 297     }
 298
 299     args_.push_back(std::move(tmp));
 300
 301     for (std::size_t j = 0; j < arglists.size(); j++) {
 302       if (ctr[j] + 1 < arglists[j].size()) {
 303         ++ctr[j];
 304         break;
 305       }
 306       ctr[j] = 0;
 307     }
 308   }
 309   return this;
 310 }
 311
 312 Benchmark* Benchmark::ArgName(const std::string& name) {
 313   CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
 314   arg_names_ = {name};
 315   return this;
 316 }
 317
 318 Benchmark* Benchmark::ArgNames(const std::vector<std::string>& names) {
 319   CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(names.size()));
 320   arg_names_ = names;
 321   return this;
 322 }
 323
 324 Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) {
 325   CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
 326   CHECK_GE(start, 0);
 327   CHECK_LE(start, limit);
 328   for (int64_t arg = start; arg <= limit; arg += step) {
 329     args_.push_back({arg});
 330   }
 331   return this;
 332 }
 333
 334 Benchmark* Benchmark::Args(const std::vector<int64_t>& args) {
 335   CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(args.size()));
 336   args_.push_back(args);
 337   return this;
 338 }
 339
 340 Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) {
 341   custom_arguments(this);
 342   return this;
 343 }
 344
 345 Benchmark* Benchmark::RangeMultiplier(int multiplier) {
 346   CHECK(multiplier > 1);
 347   range_multiplier_ = multiplier;
 348   return this;
 349 }
 350
 351 Benchmark* Benchmark::MinTime(double t) {
 352   CHECK(t > 0.0);
 353   CHECK(iterations_ == 0);
 354   min_time_ = t;
 355   return this;
 356 }
 357
 358 Benchmark* Benchmark::Iterations(size_t n) {
 359   CHECK(n > 0);
 360   CHECK(IsZero(min_time_));
 361   iterations_ = n;
 362   return this;
 363 }
 364
 365 Benchmark* Benchmark::Repetitions(int n) {
 366   CHECK(n > 0);
 367   repetitions_ = n;
 368   return this;
 369 }
 370
 371 Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
 372   report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default;
 373   return this;
 374 }
 375
 376 Benchmark* Benchmark::UseRealTime() {
 377   CHECK(!use_manual_time_)
 378       << "Cannot set UseRealTime and UseManualTime simultaneously.";
 379   use_real_time_ = true;
 380   return this;
 381 }
 382
 383 Benchmark* Benchmark::UseManualTime() {
 384   CHECK(!use_real_time_)
 385       << "Cannot set UseRealTime and UseManualTime simultaneously.";
 386   use_manual_time_ = true;
 387   return this;
 388 }
 389
 390 Benchmark* Benchmark::Complexity(BigO complexity) {
 391   complexity_ = complexity;
 392   return this;
 393 }
 394
 395 Benchmark* Benchmark::Complexity(BigOFunc* complexity) {
 396   complexity_lambda_ = complexity;
 397   complexity_ = oLambda;
 398   return this;
 399 }
 400
 401 Benchmark* Benchmark::ComputeStatistics(std::string name,
 402                                         StatisticsFunc* statistics) {
 403   statistics_.emplace_back(name, statistics);
 404   return this;
 405 }
 406
 407 Benchmark* Benchmark::Threads(int t) {
 408   CHECK_GT(t, 0);
 409   thread_counts_.push_back(t);
 410   return this;
 411 }
 412
 413 Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) {
 414   CHECK_GT(min_threads, 0);
 415   CHECK_GE(max_threads, min_threads);
 416
 417   AddRange(&thread_counts_, min_threads, max_threads, 2);
 418   return this;
 419 }
 420
 421 Benchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads,
 422                                        int stride) {
 423   CHECK_GT(min_threads, 0);
 424   CHECK_GE(max_threads, min_threads);
 425   CHECK_GE(stride, 1);
 426
 427   for (auto i = min_threads; i < max_threads; i += stride) {
 428     thread_counts_.push_back(i);
 429   }
 430   thread_counts_.push_back(max_threads);
 431   return this;
 432 }
 433
 434 Benchmark* Benchmark::ThreadPerCpu() {
 435   thread_counts_.push_back(CPUInfo::Get().num_cpus);
 436   return this;
 437 }
 438
 439 void Benchmark::SetName(const char* name) { name_ = name; }
 440
 441 int Benchmark::ArgsCnt() const {
 442   if (args_.empty()) {
 443     if (arg_names_.empty()) return -1;
 444     return static_cast<int>(arg_names_.size());
 445   }
 446   return static_cast<int>(args_.front().size());
 447 }
 448
 449 //=============================================================================//
 450 //                            FunctionBenchmark
 451 //=============================================================================//
 452
 453 void FunctionBenchmark::Run(State& st) { func_(st); }
 454
 455 }  // end namespace internal
 456
 457 void ClearRegisteredBenchmarks() {
 458   internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks();
 459 }
 460
 461 }  // end namespace benchmark