base/test/launcher/test_results_tracker.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/test_results_tracker.h"
   6
   7 #include "base/base64.h"
   8 #include "base/command_line.h"
   9 #include "base/files/file_path.h"
  10 #include "base/files/file_util.h"
  11 #include "base/format_macros.h"
  12 #include "base/json/json_file_value_serializer.h"
  13 #include "base/json/string_escape.h"
  14 #include "base/logging.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/test/launcher/test_launcher.h"
  18 #include "base/values.h"
  19
  20 namespace base {
  21
  22 namespace {
  23
  24 // The default output file for XML output.
  25 const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
  26     "test_detail.xml");
  27
  28 // Utility function to print a list of test names. Uses iterator to be
  29 // compatible with different containers, like vector and set.
  30 template<typename InputIterator>
  31 void PrintTests(InputIterator first,
  32                 InputIterator last,
  33                 const std::string& description) {
  34   size_t count = std::distance(first, last);
  35   if (count == 0)
  36     return;
  37
  38   fprintf(stdout,
  39           "%" PRIuS " test%s %s:\n",
  40           count,
  41           count != 1 ? "s" : "",
  42           description.c_str());
  43   for (InputIterator i = first; i != last; ++i)
  44     fprintf(stdout, "    %s\n", (*i).c_str());
  45   fflush(stdout);
  46 }
  47
  48 std::string TestNameWithoutDisabledPrefix(const std::string& test_name) {
  49   std::string test_name_no_disabled(test_name);
  50   ReplaceSubstringsAfterOffset(&test_name_no_disabled, 0, "DISABLED_", "");
  51   return test_name_no_disabled;
  52 }
  53
  54 }  // namespace
  55
  56 TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) {
  57 }
  58
  59 TestResultsTracker::~TestResultsTracker() {
  60   DCHECK(thread_checker_.CalledOnValidThread());
  61
  62   if (!out_)
  63     return;
  64   fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
  65   fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\""
  66           " disabled=\"\" errors=\"\" time=\"\">\n");
  67
  68   // Maps test case names to test results.
  69   typedef std::map<std::string, std::vector<TestResult> > TestCaseMap;
  70   TestCaseMap test_case_map;
  71
  72   for (PerIterationData::ResultsMap::iterator i =
  73            per_iteration_data_[iteration_].results.begin();
  74        i != per_iteration_data_[iteration_].results.end();
  75        ++i) {
  76     // Use the last test result as the final one.
  77     TestResult result = i->second.test_results.back();
  78     test_case_map[result.GetTestCaseName()].push_back(result);
  79   }
  80   for (TestCaseMap::iterator i = test_case_map.begin();
  81        i != test_case_map.end();
  82        ++i) {
  83     fprintf(out_, "  <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\""
  84             " disabled=\"\" errors=\"\" time=\"\">\n",
  85             i->first.c_str(), i->second.size());
  86     for (size_t j = 0; j < i->second.size(); ++j) {
  87       const TestResult& result = i->second[j];
  88       fprintf(out_, "    <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
  89               " classname=\"%s\">\n",
  90               result.GetTestName().c_str(),
  91               result.elapsed_time.InSecondsF(),
  92               result.GetTestCaseName().c_str());
  93       if (result.status != TestResult::TEST_SUCCESS)
  94         fprintf(out_, "      <failure message=\"\" type=\"\"></failure>\n");
  95       fprintf(out_, "    </testcase>\n");
  96     }
  97     fprintf(out_, "  </testsuite>\n");
  98   }
  99   fprintf(out_, "</testsuites>\n");
 100   fclose(out_);
 101 }
 102
 103 bool TestResultsTracker::Init(const CommandLine& command_line) {
 104   DCHECK(thread_checker_.CalledOnValidThread());
 105
 106   // Prevent initializing twice.
 107   if (out_) {
 108     NOTREACHED();
 109     return false;
 110   }
 111
 112   if (!command_line.HasSwitch(kGTestOutputFlag))
 113     return true;
 114
 115   std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
 116   size_t colon_pos = flag.find(':');
 117   FilePath path;
 118   if (colon_pos != std::string::npos) {
 119     FilePath flag_path =
 120         command_line.GetSwitchValuePath(kGTestOutputFlag);
 121     FilePath::StringType path_string = flag_path.value();
 122     path = FilePath(path_string.substr(colon_pos + 1));
 123     // If the given path ends with '/', consider it is a directory.
 124     // Note: This does NOT check that a directory (or file) actually exists
 125     // (the behavior is same as what gtest does).
 126     if (path.EndsWithSeparator()) {
 127       FilePath executable = command_line.GetProgram().BaseName();
 128       path = path.Append(executable.ReplaceExtension(
 129                              FilePath::StringType(FILE_PATH_LITERAL("xml"))));
 130     }
 131   }
 132   if (path.value().empty())
 133     path = FilePath(kDefaultOutputFile);
 134   FilePath dir_name = path.DirName();
 135   if (!DirectoryExists(dir_name)) {
 136     LOG(WARNING) << "The output directory does not exist. "
 137                  << "Creating the directory: " << dir_name.value();
 138     // Create the directory if necessary (because the gtest does the same).
 139     if (!base::CreateDirectory(dir_name)) {
 140       LOG(ERROR) << "Failed to created directory " << dir_name.value();
 141       return false;
 142     }
 143   }
 144   out_ = OpenFile(path, "w");
 145   if (!out_) {
 146     LOG(ERROR) << "Cannot open output file: "
 147                << path.value() << ".";
 148     return false;
 149   }
 150
 151   return true;
 152 }
 153
 154 void TestResultsTracker::OnTestIterationStarting() {
 155   DCHECK(thread_checker_.CalledOnValidThread());
 156
 157   // Start with a fresh state for new iteration.
 158   iteration_++;
 159   per_iteration_data_.push_back(PerIterationData());
 160 }
 161
 162 void TestResultsTracker::AddTest(const std::string& test_name) {
 163   // Record disabled test names without DISABLED_ prefix so that they are easy
 164   // to compare with regular test names, e.g. before or after disabling.
 165   all_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
 166 }
 167
 168 void TestResultsTracker::AddDisabledTest(const std::string& test_name) {
 169   // Record disabled test names without DISABLED_ prefix so that they are easy
 170   // to compare with regular test names, e.g. before or after disabling.
 171   disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
 172 }
 173
 174 void TestResultsTracker::AddTestResult(const TestResult& result) {
 175   DCHECK(thread_checker_.CalledOnValidThread());
 176
 177   per_iteration_data_[iteration_].results[
 178       result.full_name].test_results.push_back(result);
 179 }
 180
 181 void TestResultsTracker::PrintSummaryOfCurrentIteration() const {
 182   TestStatusMap tests_by_status(GetTestStatusMapForCurrentIteration());
 183
 184   PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
 185              tests_by_status[TestResult::TEST_FAILURE].end(),
 186              "failed");
 187   PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
 188              tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
 189              "failed on exit");
 190   PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
 191              tests_by_status[TestResult::TEST_TIMEOUT].end(),
 192              "timed out");
 193   PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
 194              tests_by_status[TestResult::TEST_CRASH].end(),
 195              "crashed");
 196   PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
 197              tests_by_status[TestResult::TEST_SKIPPED].end(),
 198              "skipped");
 199   PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
 200              tests_by_status[TestResult::TEST_UNKNOWN].end(),
 201              "had unknown result");
 202 }
 203
 204 void TestResultsTracker::PrintSummaryOfAllIterations() const {
 205   DCHECK(thread_checker_.CalledOnValidThread());
 206
 207   TestStatusMap tests_by_status(GetTestStatusMapForAllIterations());
 208
 209   fprintf(stdout, "Summary of all test iterations:\n");
 210   fflush(stdout);
 211
 212   PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
 213              tests_by_status[TestResult::TEST_FAILURE].end(),
 214              "failed");
 215   PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
 216              tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
 217              "failed on exit");
 218   PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
 219              tests_by_status[TestResult::TEST_TIMEOUT].end(),
 220              "timed out");
 221   PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
 222              tests_by_status[TestResult::TEST_CRASH].end(),
 223              "crashed");
 224   PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
 225              tests_by_status[TestResult::TEST_SKIPPED].end(),
 226              "skipped");
 227   PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
 228              tests_by_status[TestResult::TEST_UNKNOWN].end(),
 229              "had unknown result");
 230
 231   fprintf(stdout, "End of the summary.\n");
 232   fflush(stdout);
 233 }
 234
 235 void TestResultsTracker::AddGlobalTag(const std::string& tag) {
 236   global_tags_.insert(tag);
 237 }
 238
 239 bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const {
 240   scoped_ptr<DictionaryValue> summary_root(new DictionaryValue);
 241
 242   scoped_ptr<ListValue> global_tags(new ListValue);
 243   for (const auto& global_tag : global_tags_) {
 244     global_tags->AppendString(global_tag);
 245   }
 246   summary_root->Set("global_tags", global_tags.Pass());
 247
 248   scoped_ptr<ListValue> all_tests(new ListValue);
 249   for (const auto& test : all_tests_) {
 250     all_tests->AppendString(test);
 251   }
 252   summary_root->Set("all_tests", all_tests.Pass());
 253
 254   scoped_ptr<ListValue> disabled_tests(new ListValue);
 255   for (const auto& disabled_test : disabled_tests_) {
 256     disabled_tests->AppendString(disabled_test);
 257   }
 258   summary_root->Set("disabled_tests", disabled_tests.Pass());
 259
 260   scoped_ptr<ListValue> per_iteration_data(new ListValue);
 261
 262   for (int i = 0; i <= iteration_; i++) {
 263     scoped_ptr<DictionaryValue> current_iteration_data(new DictionaryValue);
 264
 265     for (PerIterationData::ResultsMap::const_iterator j =
 266              per_iteration_data_[i].results.begin();
 267          j != per_iteration_data_[i].results.end();
 268          ++j) {
 269       scoped_ptr<ListValue> test_results(new ListValue);
 270
 271       for (size_t k = 0; k < j->second.test_results.size(); k++) {
 272         const TestResult& test_result = j->second.test_results[k];
 273
 274         scoped_ptr<DictionaryValue> test_result_value(new DictionaryValue);
 275
 276         test_result_value->SetString("status", test_result.StatusAsString());
 277         test_result_value->SetInteger(
 278             "elapsed_time_ms",
 279             static_cast<int>(test_result.elapsed_time.InMilliseconds()));
 280
 281         // There are no guarantees about character encoding of the output
 282         // snippet. Escape it and record whether it was losless.
 283         // It's useful to have the output snippet as string in the summary
 284         // for easy viewing.
 285         std::string escaped_output_snippet;
 286         bool losless_snippet = EscapeJSONString(
 287             test_result.output_snippet, false, &escaped_output_snippet);
 288         test_result_value->SetString("output_snippet",
 289                                      escaped_output_snippet);
 290         test_result_value->SetBoolean("losless_snippet", losless_snippet);
 291
 292         // Also include the raw version (base64-encoded so that it can be safely
 293         // JSON-serialized - there are no guarantees about character encoding
 294         // of the snippet). This can be very useful piece of information when
 295         // debugging a test failure related to character encoding.
 296         std::string base64_output_snippet;
 297         Base64Encode(test_result.output_snippet, &base64_output_snippet);
 298         test_result_value->SetString("output_snippet_base64",
 299                                      base64_output_snippet);
 300         test_results->Append(test_result_value.Pass());
 301       }
 302
 303       current_iteration_data->SetWithoutPathExpansion(j->first,
 304                                                       test_results.Pass());
 305     }
 306     per_iteration_data->Append(current_iteration_data.Pass());
 307     summary_root->Set("per_iteration_data", per_iteration_data.Pass());
 308   }
 309
 310   JSONFileValueSerializer serializer(path);
 311   return serializer.Serialize(*summary_root);
 312 }
 313
 314 TestResultsTracker::TestStatusMap
 315     TestResultsTracker::GetTestStatusMapForCurrentIteration() const {
 316   TestStatusMap tests_by_status;
 317   GetTestStatusForIteration(iteration_, &tests_by_status);
 318   return tests_by_status;
 319 }
 320
 321 TestResultsTracker::TestStatusMap
 322     TestResultsTracker::GetTestStatusMapForAllIterations() const {
 323   TestStatusMap tests_by_status;
 324   for (int i = 0; i <= iteration_; i++)
 325     GetTestStatusForIteration(i, &tests_by_status);
 326   return tests_by_status;
 327 }
 328
 329 void TestResultsTracker::GetTestStatusForIteration(
 330     int iteration, TestStatusMap* map) const {
 331   for (PerIterationData::ResultsMap::const_iterator j =
 332            per_iteration_data_[iteration].results.begin();
 333        j != per_iteration_data_[iteration].results.end();
 334        ++j) {
 335     // Use the last test result as the final one.
 336     const TestResult& result = j->second.test_results.back();
 337     (*map)[result.status].insert(result.full_name);
 338   }
 339 }
 340
 341 TestResultsTracker::AggregateTestResult::AggregateTestResult() {
 342 }
 343
 344 TestResultsTracker::AggregateTestResult::~AggregateTestResult() {
 345 }
 346
 347 TestResultsTracker::PerIterationData::PerIterationData() {
 348 }
 349
 350 TestResultsTracker::PerIterationData::~PerIterationData() {
 351 }
 352
 353 }  // namespace base