third-party/benchmark/src/json_reporter.cc

   1 // Copyright 2015 Google Inc. All rights reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 #include <algorithm>
  16 #include <cmath>
  17 #include <cstdint>
  18 #include <iomanip>  // for setprecision
  19 #include <iostream>
  20 #include <limits>
  21 #include <string>
  22 #include <tuple>
  23 #include <vector>
  24
  25 #include "benchmark/benchmark.h"
  26 #include "complexity.h"
  27 #include "string_util.h"
  28 #include "timers.h"
  29
  30 namespace benchmark {
  31 namespace {
  32
  33 std::string StrEscape(const std::string& s) {
  34   std::string tmp;
  35   tmp.reserve(s.size());
  36   for (char c : s) {
  37     switch (c) {
  38       case '\b':
  39         tmp += "\\b";
  40         break;
  41       case '\f':
  42         tmp += "\\f";
  43         break;
  44       case '\n':
  45         tmp += "\\n";
  46         break;
  47       case '\r':
  48         tmp += "\\r";
  49         break;
  50       case '\t':
  51         tmp += "\\t";
  52         break;
  53       case '\\':
  54         tmp += "\\\\";
  55         break;
  56       case '"':
  57         tmp += "\\\"";
  58         break;
  59       default:
  60         tmp += c;
  61         break;
  62     }
  63   }
  64   return tmp;
  65 }
  66
  67 std::string FormatKV(std::string const& key, std::string const& value) {
  68   return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(),
  69                    StrEscape(value).c_str());
  70 }
  71
  72 std::string FormatKV(std::string const& key, const char* value) {
  73   return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(),
  74                    StrEscape(value).c_str());
  75 }
  76
  77 std::string FormatKV(std::string const& key, bool value) {
  78   return StrFormat("\"%s\": %s", StrEscape(key).c_str(),
  79                    value ? "true" : "false");
  80 }
  81
  82 std::string FormatKV(std::string const& key, int64_t value) {
  83   std::stringstream ss;
  84   ss << '"' << StrEscape(key) << "\": " << value;
  85   return ss.str();
  86 }
  87
  88 std::string FormatKV(std::string const& key, double value) {
  89   std::stringstream ss;
  90   ss << '"' << StrEscape(key) << "\": ";
  91
  92   if (std::isnan(value))
  93     ss << (value < 0 ? "-" : "") << "NaN";
  94   else if (std::isinf(value))
  95     ss << (value < 0 ? "-" : "") << "Infinity";
  96   else {
  97     const auto max_digits10 =
  98         std::numeric_limits<decltype(value)>::max_digits10;
  99     const auto max_fractional_digits10 = max_digits10 - 1;
 100     ss << std::scientific << std::setprecision(max_fractional_digits10)
 101        << value;
 102   }
 103   return ss.str();
 104 }
 105
 106 int64_t RoundDouble(double v) { return std::lround(v); }
 107
 108 }  // end namespace
 109
 110 bool JSONReporter::ReportContext(const Context& context) {
 111   std::ostream& out = GetOutputStream();
 112
 113   out << "{\n";
 114   std::string inner_indent(2, ' ');
 115
 116   // Open context block and print context information.
 117   out << inner_indent << "\"context\": {\n";
 118   std::string indent(4, ' ');
 119
 120   std::string walltime_value = LocalDateTimeString();
 121   out << indent << FormatKV("date", walltime_value) << ",\n";
 122
 123   out << indent << FormatKV("host_name", context.sys_info.name) << ",\n";
 124
 125   if (Context::executable_name) {
 126     out << indent << FormatKV("executable", Context::executable_name) << ",\n";
 127   }
 128
 129   CPUInfo const& info = context.cpu_info;
 130   out << indent << FormatKV("num_cpus", static_cast<int64_t>(info.num_cpus))
 131       << ",\n";
 132   out << indent
 133       << FormatKV("mhz_per_cpu",
 134                   RoundDouble(info.cycles_per_second / 1000000.0))
 135       << ",\n";
 136   if (CPUInfo::Scaling::UNKNOWN != info.scaling) {
 137     out << indent
 138         << FormatKV("cpu_scaling_enabled",
 139                     info.scaling == CPUInfo::Scaling::ENABLED ? true : false)
 140         << ",\n";
 141   }
 142
 143   out << indent << "\"caches\": [\n";
 144   indent = std::string(6, ' ');
 145   std::string cache_indent(8, ' ');
 146   for (size_t i = 0; i < info.caches.size(); ++i) {
 147     auto& CI = info.caches[i];
 148     out << indent << "{\n";
 149     out << cache_indent << FormatKV("type", CI.type) << ",\n";
 150     out << cache_indent << FormatKV("level", static_cast<int64_t>(CI.level))
 151         << ",\n";
 152     out << cache_indent << FormatKV("size", static_cast<int64_t>(CI.size))
 153         << ",\n";
 154     out << cache_indent
 155         << FormatKV("num_sharing", static_cast<int64_t>(CI.num_sharing))
 156         << "\n";
 157     out << indent << "}";
 158     if (i != info.caches.size() - 1) out << ",";
 159     out << "\n";
 160   }
 161   indent = std::string(4, ' ');
 162   out << indent << "],\n";
 163   out << indent << "\"load_avg\": [";
 164   for (auto it = info.load_avg.begin(); it != info.load_avg.end();) {
 165     out << *it++;
 166     if (it != info.load_avg.end()) out << ",";
 167   }
 168   out << "],\n";
 169
 170   out << indent << FormatKV("library_version", GetBenchmarkVersion());
 171   out << ",\n";
 172
 173 #if defined(NDEBUG)
 174   const char build_type[] = "release";
 175 #else
 176   const char build_type[] = "debug";
 177 #endif
 178   out << indent << FormatKV("library_build_type", build_type);
 179   out << ",\n";
 180
 181   // NOTE: our json schema is not strictly tied to the library version!
 182   out << indent << FormatKV("json_schema_version", int64_t(1));
 183
 184   std::map<std::string, std::string>* global_context =
 185       internal::GetGlobalContext();
 186
 187   if (global_context != nullptr) {
 188     for (const auto& kv : *global_context) {
 189       out << ",\n";
 190       out << indent << FormatKV(kv.first, kv.second);
 191     }
 192   }
 193   out << "\n";
 194
 195   // Close context block and open the list of benchmarks.
 196   out << inner_indent << "},\n";
 197   out << inner_indent << "\"benchmarks\": [\n";
 198   return true;
 199 }
 200
 201 void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
 202   if (reports.empty()) {
 203     return;
 204   }
 205   std::string indent(4, ' ');
 206   std::ostream& out = GetOutputStream();
 207   if (!first_report_) {
 208     out << ",\n";
 209   }
 210   first_report_ = false;
 211
 212   for (auto it = reports.begin(); it != reports.end(); ++it) {
 213     out << indent << "{\n";
 214     PrintRunData(*it);
 215     out << indent << '}';
 216     auto it_cp = it;
 217     if (++it_cp != reports.end()) {
 218       out << ",\n";
 219     }
 220   }
 221 }
 222
 223 void JSONReporter::Finalize() {
 224   // Close the list of benchmarks and the top level object.
 225   GetOutputStream() << "\n  ]\n}\n";
 226 }
 227
 228 void JSONReporter::PrintRunData(Run const& run) {
 229   std::string indent(6, ' ');
 230   std::ostream& out = GetOutputStream();
 231   out << indent << FormatKV("name", run.benchmark_name()) << ",\n";
 232   out << indent << FormatKV("family_index", run.family_index) << ",\n";
 233   out << indent
 234       << FormatKV("per_family_instance_index", run.per_family_instance_index)
 235       << ",\n";
 236   out << indent << FormatKV("run_name", run.run_name.str()) << ",\n";
 237   out << indent << FormatKV("run_type", [&run]() -> const char* {
 238     switch (run.run_type) {
 239       case BenchmarkReporter::Run::RT_Iteration:
 240         return "iteration";
 241       case BenchmarkReporter::Run::RT_Aggregate:
 242         return "aggregate";
 243     }
 244     BENCHMARK_UNREACHABLE();
 245   }()) << ",\n";
 246   out << indent << FormatKV("repetitions", run.repetitions) << ",\n";
 247   if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) {
 248     out << indent << FormatKV("repetition_index", run.repetition_index)
 249         << ",\n";
 250   }
 251   out << indent << FormatKV("threads", run.threads) << ",\n";
 252   if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) {
 253     out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n";
 254     out << indent << FormatKV("aggregate_unit", [&run]() -> const char* {
 255       switch (run.aggregate_unit) {
 256         case StatisticUnit::kTime:
 257           return "time";
 258         case StatisticUnit::kPercentage:
 259           return "percentage";
 260       }
 261       BENCHMARK_UNREACHABLE();
 262     }()) << ",\n";
 263   }
 264   if (internal::SkippedWithError == run.skipped) {
 265     out << indent << FormatKV("error_occurred", true) << ",\n";
 266     out << indent << FormatKV("error_message", run.skip_message) << ",\n";
 267   } else if (internal::SkippedWithMessage == run.skipped) {
 268     out << indent << FormatKV("skipped", true) << ",\n";
 269     out << indent << FormatKV("skip_message", run.skip_message) << ",\n";
 270   }
 271   if (!run.report_big_o && !run.report_rms) {
 272     out << indent << FormatKV("iterations", run.iterations) << ",\n";
 273     if (run.run_type != Run::RT_Aggregate ||
 274         run.aggregate_unit == StatisticUnit::kTime) {
 275       out << indent << FormatKV("real_time", run.GetAdjustedRealTime())
 276           << ",\n";
 277       out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime());
 278     } else {
 279       assert(run.aggregate_unit == StatisticUnit::kPercentage);
 280       out << indent << FormatKV("real_time", run.real_accumulated_time)
 281           << ",\n";
 282       out << indent << FormatKV("cpu_time", run.cpu_accumulated_time);
 283     }
 284     out << ",\n"
 285         << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
 286   } else if (run.report_big_o) {
 287     out << indent << FormatKV("cpu_coefficient", run.GetAdjustedCPUTime())
 288         << ",\n";
 289     out << indent << FormatKV("real_coefficient", run.GetAdjustedRealTime())
 290         << ",\n";
 291     out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n";
 292     out << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
 293   } else if (run.report_rms) {
 294     out << indent << FormatKV("rms", run.GetAdjustedCPUTime());
 295   }
 296
 297   for (auto& c : run.counters) {
 298     out << ",\n" << indent << FormatKV(c.first, c.second);
 299   }
 300
 301   if (run.memory_result) {
 302     const MemoryManager::Result memory_result = *run.memory_result;
 303     out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter);
 304     out << ",\n"
 305         << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used);
 306
 307     auto report_if_present = [&out, &indent](const std::string& label,
 308                                              int64_t val) {
 309       if (val != MemoryManager::TombstoneValue)
 310         out << ",\n" << indent << FormatKV(label, val);
 311     };
 312
 313     report_if_present("total_allocated_bytes",
 314                       memory_result.total_allocated_bytes);
 315     report_if_present("net_heap_growth", memory_result.net_heap_growth);
 316   }
 317
 318   if (!run.report_label.empty()) {
 319     out << ",\n" << indent << FormatKV("label", run.report_label);
 320   }
 321   out << '\n';
 322 }
 323
 324 const int64_t MemoryManager::TombstoneValue =
 325     std::numeric_limits<int64_t>::max();
 326
 327 }  // end namespace benchmark