chrome/test/perf/dom_checker_uitest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <list>
   6 #include <set>
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/command_line.h"
  11 #include "base/file_util.h"
  12 #include "base/files/file_path.h"
  13 #include "base/json/json_string_value_serializer.h"
  14 #include "base/memory/scoped_ptr.h"
  15 #include "base/path_service.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/strings/string_util.h"
  18 #include "base/strings/utf_string_conversions.h"
  19 #include "base/test/test_timeouts.h"
  20 #include "base/values.h"
  21 #include "chrome/common/chrome_paths.h"
  22 #include "chrome/common/chrome_switches.h"
  23 #include "chrome/test/automation/tab_proxy.h"
  24 #include "chrome/test/ui/ui_test.h"
  25 #include "net/base/net_util.h"
  26 #include "url/gurl.h"
  27
  28 namespace {
  29
  30 const base::FilePath::CharType kBaseUrl[] =
  31     FILE_PATH_LITERAL("http://localhost:8000/");
  32
  33 const base::FilePath::CharType kTestDirectory[] =
  34     FILE_PATH_LITERAL("dom_checker/");
  35
  36 const base::FilePath::CharType kStartFile[] =
  37     FILE_PATH_LITERAL("dom_checker.html");
  38
  39 const char kRunDomCheckerTest[] = "run-dom-checker-test";
  40
  41 class DomCheckerTest : public UITest {
  42  public:
  43   typedef std::list<std::string> ResultsList;
  44   typedef std::set<std::string> ResultsSet;
  45
  46   DomCheckerTest() {
  47     dom_automation_enabled_ = true;
  48     enable_file_cookies_ = false;
  49     show_window_ = true;
  50     launch_arguments_.AppendSwitch(switches::kDisablePopupBlocking);
  51   }
  52
  53   void RunTest(bool use_http, ResultsList* new_passes,
  54                ResultsList* new_failures) {
  55     int test_count = 0;
  56     ResultsSet expected_failures, current_failures;
  57
  58     std::string failures_file = use_http ?
  59 #if defined(OS_MACOSX)
  60         "expected_failures-http.txt" : "expected_failures_mac-file.txt";
  61 #elif defined(OS_LINUX)
  62         "expected_failures-http.txt" : "expected_failures_linux-file.txt";
  63 #elif defined(OS_WIN)
  64         "expected_failures-http.txt" : "expected_failures_win-file.txt";
  65 #else
  66         "" : "";
  67 #endif
  68
  69     ASSERT_TRUE(GetExpectedFailures(failures_file, &expected_failures));
  70
  71     RunDomChecker(use_http, &test_count, &current_failures);
  72     printf("\nTests run: %d\n", test_count);
  73
  74     // Compute the list of new passes and failures.
  75     CompareSets(current_failures, expected_failures, new_passes);
  76     CompareSets(expected_failures, current_failures, new_failures);
  77   }
  78
  79   void PrintResults(const ResultsList& new_passes,
  80                     const ResultsList& new_failures) {
  81     PrintResults(new_failures, "new tests failing", true);
  82     PrintResults(new_passes, "new tests passing", false);
  83   }
  84
  85  private:
  86   void PrintResults(const ResultsList& results, const char* message,
  87                     bool add_failure) {
  88     if (!results.empty()) {
  89       if (add_failure)
  90         ADD_FAILURE();
  91
  92       printf("%s:\n", message);
  93       ResultsList::const_iterator it = results.begin();
  94       for (; it != results.end(); ++it)
  95         printf("  %s\n", it->c_str());
  96       printf("\n");
  97     }
  98   }
  99
 100   // Find the elements of "b" that are not in "a".
 101   void CompareSets(const ResultsSet& a, const ResultsSet& b,
 102                    ResultsList* only_in_b) {
 103     ResultsSet::const_iterator it = b.begin();
 104     for (; it != b.end(); ++it) {
 105       if (a.find(*it) == a.end())
 106         only_in_b->push_back(*it);
 107     }
 108   }
 109
 110   // Return the path to the DOM checker directory on the local filesystem.
 111   base::FilePath GetDomCheckerDir() {
 112     base::FilePath test_dir;
 113     PathService::Get(chrome::DIR_TEST_DATA, &test_dir);
 114     return test_dir.AppendASCII("dom_checker");
 115   }
 116
 117   bool ReadExpectedResults(const std::string& failures_file,
 118                            std::string* results) {
 119     base::FilePath results_path = GetDomCheckerDir();
 120     results_path = results_path.AppendASCII(failures_file);
 121     return base::ReadFileToString(results_path, results);
 122   }
 123
 124   void ParseExpectedFailures(const std::string& input, ResultsSet* output) {
 125     if (input.empty())
 126       return;
 127
 128     std::vector<std::string> tokens;
 129     base::SplitString(input, '\n', &tokens);
 130
 131     std::vector<std::string>::const_iterator it = tokens.begin();
 132     for (; it != tokens.end(); ++it) {
 133       // Allow comments (lines that start with #).
 134       if (it->length() > 0 && it->at(0) != '#')
 135         output->insert(*it);
 136     }
 137   }
 138
 139   bool GetExpectedFailures(const std::string& failures_file,
 140                            ResultsSet* expected_failures) {
 141     std::string expected_failures_text;
 142     bool have_expected_results = ReadExpectedResults(failures_file,
 143                                                      &expected_failures_text);
 144     if (!have_expected_results)
 145       return false;
 146     ParseExpectedFailures(expected_failures_text, expected_failures);
 147     return true;
 148   }
 149
 150   bool WaitUntilTestCompletes(TabProxy* tab) {
 151     return WaitUntilJavaScriptCondition(
 152         tab,
 153         std::wstring(),
 154         L"window.domAutomationController.send(automation.IsDone());",
 155         TestTimeouts::large_test_timeout());
 156   }
 157
 158   bool GetTestCount(TabProxy* tab, int* test_count) {
 159     return tab->ExecuteAndExtractInt(
 160         std::wstring(),
 161         L"window.domAutomationController.send(automation.GetTestCount());",
 162         test_count);
 163   }
 164
 165   bool GetTestsFailed(TabProxy* tab, ResultsSet* tests_failed) {
 166     std::wstring json_wide;
 167     bool succeeded = tab->ExecuteAndExtractString(
 168         std::wstring(),
 169         L"window.domAutomationController.send("
 170         L"    JSON.stringify(automation.GetFailures()));",
 171         &json_wide);
 172
 173     // Note that we don't use ASSERT_TRUE here (and in some other places) as it
 174     // doesn't work inside a function with a return type other than void.
 175     EXPECT_TRUE(succeeded);
 176     if (!succeeded)
 177       return false;
 178
 179     std::string json = base::WideToUTF8(json_wide);
 180     JSONStringValueSerializer deserializer(json);
 181     scoped_ptr<base::Value> value(deserializer.Deserialize(NULL, NULL));
 182
 183     EXPECT_TRUE(value.get());
 184     if (!value.get())
 185       return false;
 186
 187     EXPECT_TRUE(value->IsType(base::Value::TYPE_LIST));
 188     if (!value->IsType(base::Value::TYPE_LIST))
 189       return false;
 190
 191     base::ListValue* list_value = static_cast<base::ListValue*>(value.get());
 192
 193     // The parsed JSON object will be an array of strings, each of which is a
 194     // test failure. Add those strings to the results set.
 195     base::ListValue::const_iterator it = list_value->begin();
 196     for (; it != list_value->end(); ++it) {
 197       EXPECT_TRUE((*it)->IsType(base::Value::TYPE_STRING));
 198       if ((*it)->IsType(base::Value::TYPE_STRING)) {
 199         std::string test_name;
 200         succeeded = (*it)->GetAsString(&test_name);
 201         EXPECT_TRUE(succeeded);
 202         if (succeeded)
 203           tests_failed->insert(test_name);
 204       }
 205     }
 206
 207     return true;
 208   }
 209
 210   void RunDomChecker(bool use_http, int* test_count, ResultsSet* tests_failed) {
 211     GURL test_url;
 212     base::FilePath::StringType start_file(kStartFile);
 213     if (use_http) {
 214       base::FilePath::StringType test_directory(kTestDirectory);
 215       base::FilePath::StringType url_string(kBaseUrl);
 216       url_string.append(test_directory);
 217       url_string.append(start_file);
 218       test_url = GURL(url_string);
 219     } else {
 220       base::FilePath test_path = GetDomCheckerDir();
 221       test_path = test_path.Append(start_file);
 222       test_url = net::FilePathToFileURL(test_path);
 223     }
 224
 225     scoped_refptr<TabProxy> tab(GetActiveTab());
 226     ASSERT_TRUE(tab.get());
 227     ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS, tab->NavigateToURL(test_url));
 228
 229     // Wait for the test to finish.
 230     ASSERT_TRUE(WaitUntilTestCompletes(tab.get()));
 231
 232     // Get the test results.
 233     ASSERT_TRUE(GetTestCount(tab.get(), test_count));
 234     ASSERT_TRUE(GetTestsFailed(tab.get(), tests_failed));
 235     ASSERT_GT(*test_count, 0);
 236   }
 237
 238   DISALLOW_COPY_AND_ASSIGN(DomCheckerTest);
 239 };
 240
 241 // Always fails, see but http://crbug.com/21321
 242 TEST_F(DomCheckerTest, DISABLED_File) {
 243   if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunDomCheckerTest))
 244     return;
 245
 246   ResultsList new_passes, new_failures;
 247   RunTest(false, &new_passes, &new_failures);
 248   PrintResults(new_passes, new_failures);
 249 }
 250
 251 // This test was previously failing because it was looking for an
 252 // expected results file that didn't exist.  Fixing that bug revealed
 253 // that the expected results weren't correct anyway.
 254 // http://crbug.com/21321
 255 TEST_F(DomCheckerTest, DISABLED_Http) {
 256   if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunDomCheckerTest))
 257     return;
 258
 259   ResultsList new_passes, new_failures;
 260   RunTest(true, &new_passes, &new_failures);
 261   PrintResults(new_passes, new_failures);
 262 }
 263
 264 }  // namespace