third-party/benchmark/test/output_test.h

   1 #ifndef TEST_OUTPUT_TEST_H
   2 #define TEST_OUTPUT_TEST_H
   3
   4 #undef NDEBUG
   5 #include <functional>
   6 #include <initializer_list>
   7 #include <memory>
   8 #include <sstream>
   9 #include <string>
  10 #include <utility>
  11 #include <vector>
  12
  13 #include "../src/re.h"
  14 #include "benchmark/benchmark.h"
  15
  16 #define CONCAT2(x, y) x##y
  17 #define CONCAT(x, y) CONCAT2(x, y)
  18
  19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
  20
  21 #define SET_SUBSTITUTIONS(...) \
  22   int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
  23
  24 enum MatchRules {
  25   MR_Default,  // Skip non-matching lines until a match is found.
  26   MR_Next,     // Match must occur on the next line.
  27   MR_Not  // No line between the current position and the next match matches
  28           // the regex
  29 };
  30
  31 struct TestCase {
  32   TestCase(std::string re, int rule = MR_Default);
  33
  34   std::string regex_str;
  35   int match_rule;
  36   std::string substituted_regex;
  37   std::shared_ptr<benchmark::Regex> regex;
  38 };
  39
  40 enum TestCaseID {
  41   TC_ConsoleOut,
  42   TC_ConsoleErr,
  43   TC_JSONOut,
  44   TC_JSONErr,
  45   TC_CSVOut,
  46   TC_CSVErr,
  47
  48   TC_NumID  // PRIVATE
  49 };
  50
  51 // Add a list of test cases to be run against the output specified by
  52 // 'ID'
  53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
  54
  55 // Add or set a list of substitutions to be performed on constructed regex's
  56 // See 'output_test_helper.cc' for a list of default substitutions.
  57 int SetSubstitutions(
  58     std::initializer_list<std::pair<std::string, std::string>> il);
  59
  60 // Run all output tests.
  61 void RunOutputTests(int argc, char* argv[]);
  62
  63 // Count the number of 'pat' substrings in the 'haystack' string.
  64 int SubstrCnt(const std::string& haystack, const std::string& pat);
  65
  66 // Run registered benchmarks with file reporter enabled, and return the content
  67 // outputted by the file reporter.
  68 std::string GetFileReporterOutput(int argc, char* argv[]);
  69
  70 // ========================================================================= //
  71 // ------------------------- Results checking ------------------------------ //
  72 // ========================================================================= //
  73
  74 // Call this macro to register a benchmark for checking its results. This
  75 // should be all that's needed. It subscribes a function to check the (CSV)
  76 // results of a benchmark. This is done only after verifying that the output
  77 // strings are really as expected.
  78 // bm_name_pattern: a name or a regex pattern which will be matched against
  79 //                  all the benchmark names. Matching benchmarks
  80 //                  will be the subject of a call to checker_function
  81 // checker_function: should be of type ResultsCheckFn (see below)
  82 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
  83   size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
  84
  85 struct Results;
  86 typedef std::function<void(Results const&)> ResultsCheckFn;
  87
  88 size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn);
  89
  90 // Class holding the results of a benchmark.
  91 // It is passed in calls to checker functions.
  92 struct Results {
  93   // the benchmark name
  94   std::string name;
  95   // the benchmark fields
  96   std::map<std::string, std::string> values;
  97
  98   Results(const std::string& n) : name(n) {}
  99
 100   int NumThreads() const;
 101
 102   double NumIterations() const;
 103
 104   typedef enum { kCpuTime, kRealTime } BenchmarkTime;
 105
 106   // get cpu_time or real_time in seconds
 107   double GetTime(BenchmarkTime which) const;
 108
 109   // get the real_time duration of the benchmark in seconds.
 110   // it is better to use fuzzy float checks for this, as the float
 111   // ASCII formatting is lossy.
 112   double DurationRealTime() const {
 113     return NumIterations() * GetTime(kRealTime);
 114   }
 115   // get the cpu_time duration of the benchmark in seconds
 116   double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }
 117
 118   // get the string for a result by name, or nullptr if the name
 119   // is not found
 120   const std::string* Get(const char* entry_name) const {
 121     auto it = values.find(entry_name);
 122     if (it == values.end()) return nullptr;
 123     return &it->second;
 124   }
 125
 126   // get a result by name, parsed as a specific type.
 127   // NOTE: for counters, use GetCounterAs instead.
 128   template <class T>
 129   T GetAs(const char* entry_name) const;
 130
 131   // counters are written as doubles, so they have to be read first
 132   // as a double, and only then converted to the asked type.
 133   template <class T>
 134   T GetCounterAs(const char* entry_name) const {
 135     double dval = GetAs<double>(entry_name);
 136     T tval = static_cast<T>(dval);
 137     return tval;
 138   }
 139 };
 140
 141 template <class T>
 142 T Results::GetAs(const char* entry_name) const {
 143   auto* sv = Get(entry_name);
 144   BM_CHECK(sv != nullptr && !sv->empty());
 145   std::stringstream ss;
 146   ss << *sv;
 147   T out;
 148   ss >> out;
 149   BM_CHECK(!ss.fail());
 150   return out;
 151 }
 152
 153 //----------------------------------
 154 // Macros to help in result checking. Do not use them with arguments causing
 155 // side-effects.
 156
 157 // clang-format off
 158
 159 #define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \
 160     CONCAT(BM_CHECK_, relationship)                                        \
 161     (entry.getfn< var_type >(var_name), (value)) << "\n"                \
 162     << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
 163     << __FILE__ << ":" << __LINE__ << ": "                              \
 164     << "expected (" << #var_type << ")" << (var_name)                   \
 165     << "=" << (entry).getfn< var_type >(var_name)                       \
 166     << " to be " #relationship " to " << (value) << "\n"
 167
 168 // check with tolerance. eps_factor is the tolerance window, which is
 169 // interpreted relative to value (eg, 0.1 means 10% of value).
 170 #define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
 171     CONCAT(BM_CHECK_FLOAT_, relationship)                                  \
 172     (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
 173     << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
 174     << __FILE__ << ":" << __LINE__ << ": "                              \
 175     << "expected (" << #var_type << ")" << (var_name)                   \
 176     << "=" << (entry).getfn< var_type >(var_name)                       \
 177     << " to be " #relationship " to " << (value) << "\n"                \
 178     << __FILE__ << ":" << __LINE__ << ": "                              \
 179     << "with tolerance of " << (eps_factor) * (value)                   \
 180     << " (" << (eps_factor)*100. << "%), "                              \
 181     << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
 182     << " (" << (((entry).getfn< var_type >(var_name) - (value))         \
 183                /                                                        \
 184                ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
 185     << "%)"
 186
 187 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
 188     CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value)
 189
 190 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
 191     CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value)
 192
 193 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \
 194     CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor)
 195
 196 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \
 197     CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
 198
 199 // clang-format on
 200
 201 // ========================================================================= //
 202 // --------------------------- Misc Utilities ------------------------------ //
 203 // ========================================================================= //
 204
 205 namespace {
 206
 207 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
 208
 209 }  //  end namespace
 210
 211 #endif  // TEST_OUTPUT_TEST_H