utils/benchmark/test/output_test.h

   1 #ifndef TEST_OUTPUT_TEST_H
   2 #define TEST_OUTPUT_TEST_H
   3
   4 #undef NDEBUG
   5 #include <initializer_list>
   6 #include <memory>
   7 #include <string>
   8 #include <utility>
   9 #include <vector>
  10 #include <functional>
  11 #include <sstream>
  12
  13 #include "../src/re.h"
  14 #include "benchmark/benchmark.h"
  15
  16 #define CONCAT2(x, y) x##y
  17 #define CONCAT(x, y) CONCAT2(x, y)
  18
  19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
  20
  21 #define SET_SUBSTITUTIONS(...) \
  22   int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
  23
  24 enum MatchRules {
  25   MR_Default,  // Skip non-matching lines until a match is found.
  26   MR_Next,     // Match must occur on the next line.
  27   MR_Not  // No line between the current position and the next match matches
  28           // the regex
  29 };
  30
  31 struct TestCase {
  32   TestCase(std::string re, int rule = MR_Default);
  33
  34   std::string regex_str;
  35   int match_rule;
  36   std::string substituted_regex;
  37   std::shared_ptr<benchmark::Regex> regex;
  38 };
  39
  40 enum TestCaseID {
  41   TC_ConsoleOut,
  42   TC_ConsoleErr,
  43   TC_JSONOut,
  44   TC_JSONErr,
  45   TC_CSVOut,
  46   TC_CSVErr,
  47
  48   TC_NumID  // PRIVATE
  49 };
  50
  51 // Add a list of test cases to be run against the output specified by
  52 // 'ID'
  53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
  54
  55 // Add or set a list of substitutions to be performed on constructed regex's
  56 // See 'output_test_helper.cc' for a list of default substitutions.
  57 int SetSubstitutions(
  58     std::initializer_list<std::pair<std::string, std::string>> il);
  59
  60 // Run all output tests.
  61 void RunOutputTests(int argc, char* argv[]);
  62
  63 // ========================================================================= //
  64 // ------------------------- Results checking ------------------------------ //
  65 // ========================================================================= //
  66
  67 // Call this macro to register a benchmark for checking its results. This
  68 // should be all that's needed. It subscribes a function to check the (CSV)
  69 // results of a benchmark. This is done only after verifying that the output
  70 // strings are really as expected.
  71 // bm_name_pattern: a name or a regex pattern which will be matched against
  72 //                  all the benchmark names. Matching benchmarks
  73 //                  will be the subject of a call to checker_function
  74 // checker_function: should be of type ResultsCheckFn (see below)
  75 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
  76     size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
  77
  78 struct Results;
  79 typedef std::function< void(Results const&) > ResultsCheckFn;
  80
  81 size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn);
  82
  83 // Class holding the results of a benchmark.
  84 // It is passed in calls to checker functions.
  85 struct Results {
  86
  87   // the benchmark name
  88   std::string name;
  89   // the benchmark fields
  90   std::map< std::string, std::string > values;
  91
  92   Results(const std::string& n) : name(n) {}
  93
  94   int NumThreads() const;
  95
  96   typedef enum { kCpuTime, kRealTime } BenchmarkTime;
  97
  98   // get cpu_time or real_time in seconds
  99   double GetTime(BenchmarkTime which) const;
 100
 101   // get the real_time duration of the benchmark in seconds.
 102   // it is better to use fuzzy float checks for this, as the float
 103   // ASCII formatting is lossy.
 104   double DurationRealTime() const {
 105     return GetAs< double >("iterations") * GetTime(kRealTime);
 106   }
 107   // get the cpu_time duration of the benchmark in seconds
 108   double DurationCPUTime() const {
 109     return GetAs< double >("iterations") * GetTime(kCpuTime);
 110   }
 111
 112   // get the string for a result by name, or nullptr if the name
 113   // is not found
 114   const std::string* Get(const char* entry_name) const {
 115     auto it = values.find(entry_name);
 116     if(it == values.end()) return nullptr;
 117     return &it->second;
 118   }
 119
 120   // get a result by name, parsed as a specific type.
 121   // NOTE: for counters, use GetCounterAs instead.
 122   template <class T>
 123   T GetAs(const char* entry_name) const;
 124
 125   // counters are written as doubles, so they have to be read first
 126   // as a double, and only then converted to the asked type.
 127   template <class T>
 128   T GetCounterAs(const char* entry_name) const {
 129     double dval = GetAs< double >(entry_name);
 130     T tval = static_cast< T >(dval);
 131     return tval;
 132   }
 133 };
 134
 135 template <class T>
 136 T Results::GetAs(const char* entry_name) const {
 137   auto *sv = Get(entry_name);
 138   CHECK(sv != nullptr && !sv->empty());
 139   std::stringstream ss;
 140   ss << *sv;
 141   T out;
 142   ss >> out;
 143   CHECK(!ss.fail());
 144   return out;
 145 }
 146
 147 //----------------------------------
 148 // Macros to help in result checking. Do not use them with arguments causing
 149 // side-effects.
 150
 151 #define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
 152     CONCAT(CHECK_, relationship)                                        \
 153     (entry.getfn< var_type >(var_name), (value)) << "\n"                \
 154     << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
 155     << __FILE__ << ":" << __LINE__ << ": "                              \
 156     << "expected (" << #var_type << ")" << (var_name)                   \
 157     << "=" << (entry).getfn< var_type >(var_name)                       \
 158     << " to be " #relationship " to " << (value) << "\n"
 159
 160 // check with tolerance. eps_factor is the tolerance window, which is
 161 // interpreted relative to value (eg, 0.1 means 10% of value).
 162 #define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
 163     CONCAT(CHECK_FLOAT_, relationship)                                  \
 164     (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
 165     << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n"     \
 166     << __FILE__ << ":" << __LINE__ << ": "                              \
 167     << "expected (" << #var_type << ")" << (var_name)                   \
 168     << "=" << (entry).getfn< var_type >(var_name)                       \
 169     << " to be " #relationship " to " << (value) << "\n"                \
 170     << __FILE__ << ":" << __LINE__ << ": "                              \
 171     << "with tolerance of " << (eps_factor) * (value)                   \
 172     << " (" << (eps_factor)*100. << "%), "                              \
 173     << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
 174     << " (" << (((entry).getfn< var_type >(var_name) - (value))         \
 175                /                                                        \
 176                ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
 177     << "%)"
 178
 179 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
 180     _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
 181
 182 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
 183     _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
 184
 185 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \
 186     _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor)
 187
 188 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \
 189     _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
 190
 191 // ========================================================================= //
 192 // --------------------------- Misc Utilities ------------------------------ //
 193 // ========================================================================= //
 194
 195 namespace {
 196
 197 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
 198
 199 }  //  end namespace
 200
 201 #endif  // TEST_OUTPUT_TEST_H