base/test/launcher/test_results_tracker.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/test_results_tracker.h"
   6
   7 #include "base/base64.h"
   8 #include "base/command_line.h"
   9 #include "base/files/file_path.h"
  10 #include "base/files/file_util.h"
  11 #include "base/format_macros.h"
  12 #include "base/json/json_file_value_serializer.h"
  13 #include "base/json/string_escape.h"
  14 #include "base/logging.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/test/launcher/test_launcher.h"
  18 #include "base/values.h"
  19
  20 namespace base {
  21
  22 namespace {
  23
  24 // The default output file for XML output.
  25 const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
  26     "test_detail.xml");
  27
  28 std::string TestNameWithoutDisabledPrefix(const std::string& test_name) {
  29   std::string test_name_no_disabled(test_name);
  30   ReplaceSubstringsAfterOffset(&test_name_no_disabled, 0, "DISABLED_", "");
  31   return test_name_no_disabled;
  32 }
  33
  34 }  // namespace
  35
  36 TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) {
  37 }
  38
  39 TestResultsTracker::~TestResultsTracker() {
  40   DCHECK(thread_checker_.CalledOnValidThread());
  41
  42   if (!out_)
  43     return;
  44   fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
  45   fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\""
  46           " disabled=\"\" errors=\"\" time=\"\">\n");
  47
  48   // Maps test case names to test results.
  49   typedef std::map<std::string, std::vector<TestResult> > TestCaseMap;
  50   TestCaseMap test_case_map;
  51
  52   for (PerIterationData::ResultsMap::iterator i =
  53            per_iteration_data_[iteration_].results.begin();
  54        i != per_iteration_data_[iteration_].results.end();
  55        ++i) {
  56     // Use the last test result as the final one.
  57     TestResult result = i->second.test_results.back();
  58     test_case_map[result.GetTestCaseName()].push_back(result);
  59   }
  60   for (TestCaseMap::iterator i = test_case_map.begin();
  61        i != test_case_map.end();
  62        ++i) {
  63     fprintf(out_, "  <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\""
  64             " disabled=\"\" errors=\"\" time=\"\">\n",
  65             i->first.c_str(), i->second.size());
  66     for (size_t j = 0; j < i->second.size(); ++j) {
  67       const TestResult& result = i->second[j];
  68       fprintf(out_, "    <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
  69               " classname=\"%s\">\n",
  70               result.GetTestName().c_str(),
  71               result.elapsed_time.InSecondsF(),
  72               result.GetTestCaseName().c_str());
  73       if (result.status != TestResult::TEST_SUCCESS)
  74         fprintf(out_, "      <failure message=\"\" type=\"\"></failure>\n");
  75       fprintf(out_, "    </testcase>\n");
  76     }
  77     fprintf(out_, "  </testsuite>\n");
  78   }
  79   fprintf(out_, "</testsuites>\n");
  80   fclose(out_);
  81 }
  82
  83 bool TestResultsTracker::Init(const CommandLine& command_line) {
  84   DCHECK(thread_checker_.CalledOnValidThread());
  85
  86   // Prevent initializing twice.
  87   if (out_) {
  88     NOTREACHED();
  89     return false;
  90   }
  91
  92   if (!command_line.HasSwitch(kGTestOutputFlag))
  93     return true;
  94
  95   std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
  96   size_t colon_pos = flag.find(':');
  97   FilePath path;
  98   if (colon_pos != std::string::npos) {
  99     FilePath flag_path =
 100         command_line.GetSwitchValuePath(kGTestOutputFlag);
 101     FilePath::StringType path_string = flag_path.value();
 102     path = FilePath(path_string.substr(colon_pos + 1));
 103     // If the given path ends with '/', consider it is a directory.
 104     // Note: This does NOT check that a directory (or file) actually exists
 105     // (the behavior is same as what gtest does).
 106     if (path.EndsWithSeparator()) {
 107       FilePath executable = command_line.GetProgram().BaseName();
 108       path = path.Append(executable.ReplaceExtension(
 109                              FilePath::StringType(FILE_PATH_LITERAL("xml"))));
 110     }
 111   }
 112   if (path.value().empty())
 113     path = FilePath(kDefaultOutputFile);
 114   FilePath dir_name = path.DirName();
 115   if (!DirectoryExists(dir_name)) {
 116     LOG(WARNING) << "The output directory does not exist. "
 117                  << "Creating the directory: " << dir_name.value();
 118     // Create the directory if necessary (because the gtest does the same).
 119     if (!base::CreateDirectory(dir_name)) {
 120       LOG(ERROR) << "Failed to created directory " << dir_name.value();
 121       return false;
 122     }
 123   }
 124   out_ = OpenFile(path, "w");
 125   if (!out_) {
 126     LOG(ERROR) << "Cannot open output file: "
 127                << path.value() << ".";
 128     return false;
 129   }
 130
 131   return true;
 132 }
 133
 134 void TestResultsTracker::OnTestIterationStarting() {
 135   DCHECK(thread_checker_.CalledOnValidThread());
 136
 137   // Start with a fresh state for new iteration.
 138   iteration_++;
 139   per_iteration_data_.push_back(PerIterationData());
 140 }
 141
 142 void TestResultsTracker::AddTest(
 143     const std::string& test_name, const std::string& file, int line) {
 144   // Record disabled test names without DISABLED_ prefix so that they are easy
 145   // to compare with regular test names, e.g. before or after disabling.
 146   all_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
 147
 148   test_locations_.insert(std::make_pair(
 149       TestNameWithoutDisabledPrefix(test_name), CodeLocation(file, line)));
 150 }
 151
 152 void TestResultsTracker::AddDisabledTest(const std::string& test_name) {
 153   // Record disabled test names without DISABLED_ prefix so that they are easy
 154   // to compare with regular test names, e.g. before or after disabling.
 155   disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
 156 }
 157
 158 void TestResultsTracker::AddTestResult(const TestResult& result) {
 159   DCHECK(thread_checker_.CalledOnValidThread());
 160
 161   per_iteration_data_[iteration_].results[
 162       result.full_name].test_results.push_back(result);
 163 }
 164
 165 void TestResultsTracker::PrintSummaryOfCurrentIteration() const {
 166   TestStatusMap tests_by_status(GetTestStatusMapForCurrentIteration());
 167
 168   PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
 169              tests_by_status[TestResult::TEST_FAILURE].end(),
 170              "failed");
 171   PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
 172              tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
 173              "failed on exit");
 174   PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
 175              tests_by_status[TestResult::TEST_TIMEOUT].end(),
 176              "timed out");
 177   PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
 178              tests_by_status[TestResult::TEST_CRASH].end(),
 179              "crashed");
 180   PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
 181              tests_by_status[TestResult::TEST_SKIPPED].end(),
 182              "skipped");
 183   PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
 184              tests_by_status[TestResult::TEST_UNKNOWN].end(),
 185              "had unknown result");
 186 }
 187
 188 void TestResultsTracker::PrintSummaryOfAllIterations() const {
 189   DCHECK(thread_checker_.CalledOnValidThread());
 190
 191   TestStatusMap tests_by_status(GetTestStatusMapForAllIterations());
 192
 193   fprintf(stdout, "Summary of all test iterations:\n");
 194   fflush(stdout);
 195
 196   PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
 197              tests_by_status[TestResult::TEST_FAILURE].end(),
 198              "failed");
 199   PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
 200              tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
 201              "failed on exit");
 202   PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
 203              tests_by_status[TestResult::TEST_TIMEOUT].end(),
 204              "timed out");
 205   PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
 206              tests_by_status[TestResult::TEST_CRASH].end(),
 207              "crashed");
 208   PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
 209              tests_by_status[TestResult::TEST_SKIPPED].end(),
 210              "skipped");
 211   PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
 212              tests_by_status[TestResult::TEST_UNKNOWN].end(),
 213              "had unknown result");
 214
 215   fprintf(stdout, "End of the summary.\n");
 216   fflush(stdout);
 217 }
 218
 219 void TestResultsTracker::AddGlobalTag(const std::string& tag) {
 220   global_tags_.insert(tag);
 221 }
 222
 223 bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const {
 224   scoped_ptr<DictionaryValue> summary_root(new DictionaryValue);
 225
 226   scoped_ptr<ListValue> global_tags(new ListValue);
 227   for (const auto& global_tag : global_tags_) {
 228     global_tags->AppendString(global_tag);
 229   }
 230   summary_root->Set("global_tags", global_tags.Pass());
 231
 232   scoped_ptr<ListValue> all_tests(new ListValue);
 233   for (const auto& test : all_tests_) {
 234     all_tests->AppendString(test);
 235   }
 236   summary_root->Set("all_tests", all_tests.Pass());
 237
 238   scoped_ptr<ListValue> disabled_tests(new ListValue);
 239   for (const auto& disabled_test : disabled_tests_) {
 240     disabled_tests->AppendString(disabled_test);
 241   }
 242   summary_root->Set("disabled_tests", disabled_tests.Pass());
 243
 244   scoped_ptr<ListValue> per_iteration_data(new ListValue);
 245
 246   for (int i = 0; i <= iteration_; i++) {
 247     scoped_ptr<DictionaryValue> current_iteration_data(new DictionaryValue);
 248
 249     for (PerIterationData::ResultsMap::const_iterator j =
 250              per_iteration_data_[i].results.begin();
 251          j != per_iteration_data_[i].results.end();
 252          ++j) {
 253       scoped_ptr<ListValue> test_results(new ListValue);
 254
 255       for (size_t k = 0; k < j->second.test_results.size(); k++) {
 256         const TestResult& test_result = j->second.test_results[k];
 257
 258         scoped_ptr<DictionaryValue> test_result_value(new DictionaryValue);
 259
 260         test_result_value->SetString("status", test_result.StatusAsString());
 261         test_result_value->SetInteger(
 262             "elapsed_time_ms",
 263             static_cast<int>(test_result.elapsed_time.InMilliseconds()));
 264
 265         bool lossless_snippet = false;
 266         if (IsStringUTF8(test_result.output_snippet)) {
 267           test_result_value->SetString(
 268               "output_snippet", test_result.output_snippet);
 269           lossless_snippet = true;
 270         } else {
 271           test_result_value->SetString(
 272               "output_snippet",
 273               "<non-UTF-8 snippet, see output_snippet_base64>");
 274         }
 275
 276         // TODO(phajdan.jr): Fix typo in JSON key (losless -> lossless)
 277         // making sure not to break any consumers of this data.
 278         test_result_value->SetBoolean("losless_snippet", lossless_snippet);
 279
 280         // Also include the raw version (base64-encoded so that it can be safely
 281         // JSON-serialized - there are no guarantees about character encoding
 282         // of the snippet). This can be very useful piece of information when
 283         // debugging a test failure related to character encoding.
 284         std::string base64_output_snippet;
 285         Base64Encode(test_result.output_snippet, &base64_output_snippet);
 286         test_result_value->SetString("output_snippet_base64",
 287                                      base64_output_snippet);
 288         test_results->Append(test_result_value.Pass());
 289       }
 290
 291       current_iteration_data->SetWithoutPathExpansion(j->first,
 292                                                       test_results.Pass());
 293     }
 294     per_iteration_data->Append(current_iteration_data.Pass());
 295     summary_root->Set("per_iteration_data", per_iteration_data.Pass());
 296   }
 297
 298   JSONFileValueSerializer serializer(path);
 299   return serializer.Serialize(*summary_root);
 300 }
 301
 302 TestResultsTracker::TestStatusMap
 303     TestResultsTracker::GetTestStatusMapForCurrentIteration() const {
 304   TestStatusMap tests_by_status;
 305   GetTestStatusForIteration(iteration_, &tests_by_status);
 306   return tests_by_status;
 307 }
 308
 309 TestResultsTracker::TestStatusMap
 310     TestResultsTracker::GetTestStatusMapForAllIterations() const {
 311   TestStatusMap tests_by_status;
 312   for (int i = 0; i <= iteration_; i++)
 313     GetTestStatusForIteration(i, &tests_by_status);
 314   return tests_by_status;
 315 }
 316
 317 void TestResultsTracker::GetTestStatusForIteration(
 318     int iteration, TestStatusMap* map) const {
 319   for (PerIterationData::ResultsMap::const_iterator j =
 320            per_iteration_data_[iteration].results.begin();
 321        j != per_iteration_data_[iteration].results.end();
 322        ++j) {
 323     // Use the last test result as the final one.
 324     const TestResult& result = j->second.test_results.back();
 325     (*map)[result.status].insert(result.full_name);
 326   }
 327 }
 328
 329 // Utility function to print a list of test names. Uses iterator to be
 330 // compatible with different containers, like vector and set.
 331 template<typename InputIterator>
 332 void TestResultsTracker::PrintTests(InputIterator first,
 333                                     InputIterator last,
 334                                     const std::string& description) const {
 335   size_t count = std::distance(first, last);
 336   if (count == 0)
 337     return;
 338
 339   fprintf(stdout,
 340           "%" PRIuS " test%s %s:\n",
 341           count,
 342           count != 1 ? "s" : "",
 343           description.c_str());
 344   for (InputIterator i = first; i != last; ++i) {
 345     fprintf(stdout,
 346             "    %s (%s:%d)\n",
 347             (*i).c_str(),
 348             test_locations_.at(*i).file.c_str(),
 349             test_locations_.at(*i).line);
 350   }
 351   fflush(stdout);
 352 }
 353
 354
 355 TestResultsTracker::AggregateTestResult::AggregateTestResult() {
 356 }
 357
 358 TestResultsTracker::AggregateTestResult::~AggregateTestResult() {
 359 }
 360
 361 TestResultsTracker::PerIterationData::PerIterationData() {
 362 }
 363
 364 TestResultsTracker::PerIterationData::~PerIterationData() {
 365 }
 366
 367 }  // namespace base