Separate Simple Backend creation from initialization.
[chromium-blink-merge.git] / chrome_frame / test / reliability / page_load_test.cc
blob4d67392d68fefb07981e456454cda13d9ba30c66
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 //
5 // This file provides reliablity tests which run for ChromeFrame.
6 //
7 // Usage:
8 // <reliability test exe> --list=file --startline=start --endline=end [...]
9 // Upon invocation, it visits each of the URLs on line numbers between start
10 // and end, inclusive, stored in the input file. The line number starts from 1.
12 // Optional Switches:
13 // --iterations=num: goes through the list of URLs constructed in usage 2 or 3
14 // num times.
15 // --memoryusage: prints out memory usage when visiting each page.
16 // --logfile=filepath: saves the visit log to the specified path.
17 // --timeout=seconds: time out as specified in seconds during each
18 // page load.
19 // --nopagedown: won't simulate page down key presses after page load.
20 // --noclearprofile: do not clear profile dir before firing up each time.
21 // --savedebuglog: save Chrome, V8, and test debug log for each page loaded.
22 #include <fstream>
23 #include <ostream>
25 #include "base/command_line.h"
26 #include "base/file_util.h"
27 #include "base/file_version_info.h"
28 #include "base/files/file_path.h"
29 #include "base/i18n/time_formatting.h"
30 #include "base/path_service.h"
31 #include "base/prefs/json_pref_store.h"
32 #include "base/prefs/pref_registry_simple.h"
33 #include "base/prefs/pref_service.h"
34 #include "base/prefs/pref_value_store.h"
35 #include "base/string_number_conversions.h"
36 #include "base/string_util.h"
37 #include "base/test/test_file_util.h"
38 #include "base/time.h"
39 #include "base/utf_string_conversions.h"
40 #include "chrome/browser/net/url_fixer_upper.h"
41 #include "chrome/browser/prefs/pref_service_mock_builder.h"
42 #include "chrome/common/automation_messages.h"
43 #include "chrome/common/chrome_constants.h"
44 #include "chrome/common/chrome_paths.h"
45 #include "chrome/common/chrome_paths_internal.h"
46 #include "chrome/common/chrome_switches.h"
47 #include "chrome/common/logging_chrome.h"
48 #include "chrome/common/pref_names.h"
49 #include "chrome/test/automation/automation_proxy.h"
50 #include "chrome/test/automation/browser_proxy.h"
51 #include "chrome/test/automation/tab_proxy.h"
52 #include "chrome/test/automation/window_proxy.h"
53 #include "chrome/test/ui/ui_test.h"
54 #include "chrome_frame/test/chrome_frame_test_utils.h"
55 #include "chrome_frame/test/ie_event_sink.h"
56 #include "chrome_frame/test/reliability/page_load_test.h"
57 #include "chrome_frame/utils.h"
58 #include "content/public/browser/browser_thread.h"
59 #include "content/public/test/test_browser_thread.h"
60 #include "net/base/net_util.h"
61 #include "testing/gmock/include/gmock/gmock.h"
62 #include "testing/gtest/include/gtest/gtest.h"
64 using testing::StrCaseEq;
65 using testing::StrCaseNe;
67 namespace {
69 // See comments at the beginning of the file for the definition of switches.
70 const char kListSwitch[] = "list";
71 const char kStartIndexSwitch[] = "startline";
72 const char kEndIndexSwitch[] = "endline";
73 const char kIterationSwitch[] = "iterations";
74 const char kContinuousLoadSwitch[] = "continuousload";
75 const char kMemoryUsageSwitch[] = "memoryusage";
76 const char kLogFileSwitch[] = "logfile";
77 const char kTimeoutSwitch[] = "timeout";
78 const char kNoPageDownSwitch[] = "nopagedown";
79 const char kNoClearProfileSwitch[] = "noclearprofile";
80 const char kSaveDebugLogSwitch[] = "savedebuglog";
82 // These are copied from v8 definitions as we cannot include them.
83 const char kV8LogFileSwitch[] = "logfile";
84 const char kV8LogFileDefaultName[] = "v8.log";
86 // String name of local chrome dll for looking up file information.
87 const wchar_t kChromeDll[] = L"chrome.dll";
89 base::FilePath g_url_file_path;
90 int32 g_start_index = 1;
91 int32 g_end_index = kint32max;
92 int32 g_iterations = 1;
93 bool g_memory_usage = false;
94 bool g_page_down = true;
95 bool g_clear_profile = true;
96 std::string g_end_url;
97 base::FilePath g_log_file_path;
98 bool g_save_debug_log = false;
99 base::FilePath g_chrome_log_path;
100 base::FilePath g_v8_log_path;
101 base::FilePath g_test_log_path;
102 bool g_stand_alone = false;
104 const int kUrlNavigationTimeoutSeconds = 20;
105 int g_timeout_seconds = kUrlNavigationTimeoutSeconds;
107 // Mocks document complete and load events.
108 class MockLoadListener : public chrome_frame_test::IEEventListener {
109 public:
110 MOCK_METHOD1(OnDocumentComplete, void (const wchar_t* url)); // NOLINT
111 MOCK_METHOD1(OnLoad, void (const wchar_t* url)); // NOLINT
112 MOCK_METHOD0(OnQuit, void ()); // NOLINT
114 private:
115 virtual void OnDocumentComplete(IDispatch* dispatch, VARIANT* url) {
116 if (url->bstrVal)
117 OnDocumentComplete(url->bstrVal);
121 ACTION_P(QuitIE, event_sink) {
122 EXPECT_HRESULT_SUCCEEDED(event_sink->CloseWebBrowser());
125 class PageLoadTest : public testing::Test {
126 public:
127 enum NavigationResult {
128 NAVIGATION_ERROR = 0,
129 NAVIGATION_SUCCESS,
132 typedef struct {
133 // These are results from the test automation that drives Chrome
134 NavigationResult result;
135 int crash_dump_count;
136 // These are stability metrics recorded by Chrome itself
137 bool browser_clean_exit;
138 int browser_launch_count;
139 int page_load_count;
140 int browser_crash_count;
141 int renderer_crash_count;
142 int plugin_crash_count;
143 } NavigationMetrics;
145 PageLoadTest() {}
147 // Accept URL as std::string here because the url may also act as a test id
148 // and needs to be logged in its original format even if invalid.
149 void NavigateToURLLogResult(const std::string& url_string,
150 std::ofstream& log_file,
151 NavigationMetrics* metrics_output) {
152 GURL url(url_string);
153 NavigationMetrics metrics = {NAVIGATION_ERROR};
154 std::ofstream test_log;
156 // Create a test log.
157 g_test_log_path = base::FilePath(FILE_PATH_LITERAL("test_log.log"));
158 test_log.open(g_test_log_path.value().c_str());
160 // Check file version info for chrome dll.
161 scoped_ptr<FileVersionInfo> file_info;
162 #if defined(OS_WIN)
163 file_info.reset(
164 FileVersionInfo::CreateFileVersionInfo(base::FilePath(kChromeDll)));
165 #elif defined(OS_LINUX) || defined(OS_MACOSX)
166 // TODO(fmeawad): the version retrieved here belongs to the test module and
167 // not the chrome binary, need to be changed to chrome binary instead.
168 file_info.reset(FileVersionInfo::CreateFileVersionInfoForCurrentModule());
169 #endif // !defined(OS_WIN)
170 std::wstring last_change = file_info->last_change();
171 test_log << "Last Change: ";
172 test_log << last_change << std::endl;
175 // Log timestamp for test start.
176 base::Time time_now = base::Time::Now();
177 double time_start = time_now.ToDoubleT();
178 test_log << "Test Start: ";
179 test_log << base::TimeFormatFriendlyDateAndTime(time_now) << std::endl;
181 HRESULT hr = E_FAIL;
183 chrome_frame_test::TimedMsgLoop message_loop;
185 // Launch IE.
186 base::win::ScopedComPtr<IWebBrowser2> web_browser2;
187 hr = chrome_frame_test::LaunchIEAsComServer(web_browser2.Receive());
188 EXPECT_HRESULT_SUCCEEDED(hr);
189 EXPECT_TRUE(web_browser2.get() != NULL);
190 web_browser2->put_Visible(VARIANT_TRUE);
192 // Log Browser Launched time.
193 time_now = base::Time::Now();
194 test_log << "browser_launched_seconds=";
195 test_log << (time_now.ToDoubleT() - time_start) << std::endl;
197 bool is_chrome_frame_navigation =
198 StartsWith(UTF8ToWide(url.spec()), kChromeProtocolPrefix, true);
200 CComObjectStack<chrome_frame_test::IEEventSink> ie_event_sink;
201 MockLoadListener load_listener;
202 // Disregard any interstitial about:blank loads.
203 EXPECT_CALL(load_listener, OnDocumentComplete(StrCaseEq(L"about:blank")))
204 .Times(testing::AnyNumber());
206 // Note that we can't compare the loaded url directly with the given url
207 // because the page may have redirected us to a different page, e.g.
208 // www.google.com -> www.google.ca.
209 if (is_chrome_frame_navigation) {
210 EXPECT_CALL(load_listener, OnDocumentComplete(testing::_));
211 EXPECT_CALL(load_listener, OnLoad(testing::_))
212 .WillOnce(QuitIE(&ie_event_sink));
213 } else {
214 EXPECT_CALL(load_listener, OnDocumentComplete(StrCaseNe(L"about:blank")))
215 .WillOnce(QuitIE(&ie_event_sink));
217 EXPECT_CALL(load_listener, OnQuit()).WillOnce(QUIT_LOOP(message_loop));
219 // Attach the sink and navigate.
220 ie_event_sink.set_listener(&load_listener);
221 ie_event_sink.Attach(web_browser2);
222 hr = ie_event_sink.Navigate(UTF8ToWide(url.spec()));
223 if (SUCCEEDED(hr)) {
224 message_loop.RunFor(base::TimeDelta::FromSeconds(g_timeout_seconds));
225 if (!message_loop.WasTimedOut())
226 metrics.result = NAVIGATION_SUCCESS;
229 // Log navigate complete time.
230 time_now = base::Time::Now();
231 test_log << "navigate_complete_seconds=";
232 test_log << (time_now.ToDoubleT() - time_start) << std::endl;
234 // Close IE.
235 ie_event_sink.set_listener(NULL);
236 ie_event_sink.Uninitialize();
237 chrome_frame_test::CloseAllIEWindows();
239 // Log end of test time.
240 time_now = base::Time::Now();
241 test_log << "total_duration_seconds=";
242 test_log << (time_now.ToDoubleT() - time_start) << std::endl;
244 // Get navigation result and metrics, and optionally write to the log file
245 // provided. The log format is:
246 // <url> <navigation_result> <browser_crash_count> <renderer_crash_count>
247 // <plugin_crash_count> <crash_dump_count> [chrome_log=<path>
248 // v8_log=<path>] crash_dump=<path>
249 if (log_file.is_open()) {
250 log_file << url_string;
251 switch (metrics.result) {
252 case NAVIGATION_ERROR:
253 log_file << " error";
254 break;
255 case NAVIGATION_SUCCESS:
256 log_file << " success";
257 break;
258 default:
259 break;
263 // Get stability metrics recorded by Chrome itself.
264 if (is_chrome_frame_navigation) {
265 GetStabilityMetrics(&metrics);
268 if (log_file.is_open()) {
269 log_file << " " << metrics.browser_crash_count \
270 // The renderer crash count is flaky due to 1183283.
271 // Ignore the count since we also catch crash by
272 // crash_dump_count.
273 << " " << 0 \
274 << " " << metrics.plugin_crash_count \
275 << " " << metrics.crash_dump_count;
278 // Close test log.
279 test_log.close();
281 if (log_file.is_open() && g_save_debug_log)
282 SaveDebugLogs(log_file);
284 // Log revision information for Chrome build under test.
285 log_file << " " << "revision=" << last_change;
287 // Get crash dumps.
288 LogOrDeleteNewCrashDumps(log_file, &metrics);
290 if (log_file.is_open()) {
291 log_file << std::endl;
294 if (metrics_output) {
295 *metrics_output = metrics;
299 void NavigateThroughURLList(std::ofstream& log_file) {
300 std::ifstream file(g_url_file_path.value().c_str());
301 ASSERT_TRUE(file.is_open());
303 // We navigate to URLs in the following order.
304 // CF -> CF -> host -> CF -> CF -> host.
305 for (int line_index = 1;
306 line_index <= g_end_index && !file.eof();
307 ++line_index) {
308 std::string url_str;
309 std::getline(file, url_str);
311 if (file.fail()) {
312 break;
315 // Every 3rd URL goes into the host browser.
316 if (line_index % 3 != 0) {
317 std::string actual_url;
318 actual_url = WideToUTF8(kChromeProtocolPrefix);
319 actual_url += url_str;
320 url_str = actual_url;
323 if (g_start_index <= line_index) {
324 NavigateToURLLogResult(url_str, log_file, NULL);
328 file.close();
331 protected:
332 virtual void SetUp() {
333 // Initialize crash_dumps_dir_path_.
334 PathService::Get(chrome::DIR_CRASH_DUMPS, &crash_dumps_dir_path_);
335 file_util::FileEnumerator enumerator(crash_dumps_dir_path_,
336 false, // not recursive
337 file_util::FileEnumerator::FILES);
338 for (base::FilePath path = enumerator.Next(); !path.value().empty();
339 path = enumerator.Next()) {
340 if (path.MatchesExtension(FILE_PATH_LITERAL(".dmp")))
341 crash_dumps_[path.BaseName()] = true;
344 if (g_clear_profile) {
345 base::FilePath user_data_dir;
346 chrome::GetChromeFrameUserDataDirectory(&user_data_dir);
347 ASSERT_TRUE(file_util::DieFileDie(user_data_dir, true));
350 SetConfigBool(kChromeFrameHeadlessMode, true);
351 SetConfigBool(kAllowUnsafeURLs, true);
354 virtual void TearDown() {
355 DeleteConfigValue(kChromeFrameHeadlessMode);
356 DeleteConfigValue(kAllowUnsafeURLs);
359 base::FilePath ConstructSavedDebugLogPath(const base::FilePath& debug_log_path,
360 int index) {
361 std::string suffix("_");
362 suffix.append(base::IntToString(index));
363 return debug_log_path.InsertBeforeExtensionASCII(suffix);
366 void SaveDebugLog(const base::FilePath& log_path, const std::wstring& log_id,
367 std::ofstream& log_file, int index) {
368 if (!log_path.empty()) {
369 base::FilePath saved_log_file_path =
370 ConstructSavedDebugLogPath(log_path, index);
371 if (file_util::Move(log_path, saved_log_file_path)) {
372 log_file << " " << log_id << "=" << saved_log_file_path.value();
377 // Rename the chrome and v8 debug log files if existing, and save the file
378 // paths in the log_file provided.
379 void SaveDebugLogs(std::ofstream& log_file) {
380 static int url_count = 1;
381 SaveDebugLog(g_chrome_log_path, L"chrome_log", log_file, url_count);
382 SaveDebugLog(g_v8_log_path, L"v8_log", log_file, url_count);
383 SaveDebugLog(g_test_log_path, L"test_log", log_file, url_count);
384 url_count++;
387 // If a log_file is provided, log the crash dump with the given path;
388 // otherwise, delete the crash dump file.
389 void LogOrDeleteCrashDump(std::ofstream& log_file,
390 base::FilePath crash_dump_file_name) {
391 base::FilePath crash_dump_file_path(crash_dumps_dir_path_);
392 crash_dump_file_path = crash_dump_file_path.Append(crash_dump_file_name);
393 base::FilePath crash_text_file_path =
394 crash_dump_file_path.ReplaceExtension(FILE_PATH_LITERAL("txt"));
396 if (log_file.is_open()) {
397 crash_dumps_[crash_dump_file_name] = true;
398 log_file << " crash_dump=" << crash_dump_file_path.value().c_str();
399 } else {
400 ASSERT_TRUE(file_util::DieFileDie(
401 crash_dump_file_path, false));
402 ASSERT_TRUE(file_util::DieFileDie(
403 crash_text_file_path, false));
407 // Check whether there are new .dmp files. Additionally, write
408 // " crash_dump=<full path name of the .dmp file>"
409 // to log_file.
410 void LogOrDeleteNewCrashDumps(std::ofstream& log_file,
411 NavigationMetrics* metrics) {
412 int num_dumps = 0;
414 file_util::FileEnumerator enumerator(crash_dumps_dir_path_,
415 false, // not recursive
416 file_util::FileEnumerator::FILES);
417 for (base::FilePath path = enumerator.Next(); !path.value().empty();
418 path = enumerator.Next()) {
419 if (path.MatchesExtension(FILE_PATH_LITERAL(".dmp")) &&
420 !crash_dumps_[path.BaseName()]) {
421 LogOrDeleteCrashDump(log_file, path.BaseName());
422 num_dumps++;
425 if (metrics)
426 metrics->crash_dump_count = num_dumps;
429 // Get a PrefService whose contents correspond to the Local State file
430 // that was saved by the app as it closed. The caller takes ownership of the
431 // returned PrefService object.
432 PrefService* GetLocalState(PrefRegistry* registry) {
433 base::FilePath path;
434 chrome::GetChromeFrameUserDataDirectory(&path);
435 PrefServiceMockBuilder builder;
436 builder.WithUserFilePrefs(
437 path,
438 JsonPrefStore::GetTaskRunnerForFile(
439 path, content::BrowserThread::GetBlockingPool()));
440 return builder.Create(registry);
443 void GetStabilityMetrics(NavigationMetrics* metrics) {
444 if (!metrics)
445 return;
446 scoped_refptr<PrefRegistrySimple> registry = new PrefRegistrySimple();
447 registry->RegisterBooleanPref(prefs::kStabilityExitedCleanly, false);
448 registry->RegisterIntegerPref(prefs::kStabilityLaunchCount, -1);
449 registry->RegisterIntegerPref(prefs::kStabilityPageLoadCount, -1);
450 registry->RegisterIntegerPref(prefs::kStabilityCrashCount, 0);
451 registry->RegisterIntegerPref(prefs::kStabilityRendererCrashCount, 0);
453 scoped_ptr<PrefService> local_state(GetLocalState(registry));
454 if (!local_state.get())
455 return;
457 metrics->browser_clean_exit =
458 local_state->GetBoolean(prefs::kStabilityExitedCleanly);
459 metrics->browser_launch_count =
460 local_state->GetInteger(prefs::kStabilityLaunchCount);
461 metrics->page_load_count =
462 local_state->GetInteger(prefs::kStabilityPageLoadCount);
463 metrics->browser_crash_count =
464 local_state->GetInteger(prefs::kStabilityCrashCount);
465 metrics->renderer_crash_count =
466 local_state->GetInteger(prefs::kStabilityRendererCrashCount);
467 // TODO(huanr)
468 metrics->plugin_crash_count = 0;
470 if (!metrics->browser_clean_exit)
471 metrics->browser_crash_count++;
474 base::FilePath GetSampleDataDir() {
475 base::FilePath test_dir;
476 PathService::Get(chrome::DIR_TEST_DATA, &test_dir);
477 test_dir = test_dir.AppendASCII("reliability");
478 test_dir = test_dir.AppendASCII("sample_pages");
479 return test_dir;
482 // The pathname of Chrome's crash dumps directory.
483 base::FilePath crash_dumps_dir_path_;
485 // The set of all the crash dumps we have seen. Each crash generates a
486 // .dmp and a .txt file in the crash dumps directory. We only store the
487 // .dmp files in this set.
489 // The set is implemented as a std::map. The key is the file name, and
490 // the value is false (the file is not in the set) or true (the file is
491 // in the set). The initial value for any key in std::map is 0 (false),
492 // which in this case means a new file is not in the set initially,
493 // exactly the semantics we want.
494 std::map<base::FilePath, bool> crash_dumps_;
497 TEST_F(PageLoadTest, IEFullTabMode_Reliability) {
498 std::ofstream log_file;
500 if (!g_log_file_path.empty()) {
501 log_file.open(g_log_file_path.value().c_str());
504 EXPECT_FALSE(g_url_file_path.empty());
506 for (int k = 0; k < g_iterations; ++k) {
507 NavigateThroughURLList(log_file);
510 log_file.close();
513 } // namespace
515 namespace {
516 void ReportHandler(const std::string& str) {
517 // Ignore report events.
521 void SetPageRange(const CommandLine& parsed_command_line) {
522 // If calling into this function, we are running as a standalone program.
523 g_stand_alone = true;
525 // Since we use --enable-dcheck for reliability tests, suppress the error
526 // dialog in the test process.
527 logging::SetLogReportHandler(ReportHandler);
529 if (parsed_command_line.HasSwitch(kStartIndexSwitch)) {
530 ASSERT_TRUE(
531 base::StringToInt(parsed_command_line.GetSwitchValueASCII(
532 kStartIndexSwitch),
533 &g_start_index));
534 ASSERT_GT(g_start_index, 0);
537 if (parsed_command_line.HasSwitch(kEndIndexSwitch)) {
538 ASSERT_TRUE(
539 base::StringToInt(parsed_command_line.GetSwitchValueASCII(
540 kEndIndexSwitch),
541 &g_end_index));
542 ASSERT_GT(g_end_index, 0);
545 ASSERT_TRUE(g_end_index >= g_start_index);
547 if (parsed_command_line.HasSwitch(kListSwitch))
548 g_url_file_path = parsed_command_line.GetSwitchValuePath(kListSwitch);
550 if (parsed_command_line.HasSwitch(kIterationSwitch)) {
551 ASSERT_TRUE(
552 base::StringToInt(parsed_command_line.GetSwitchValueASCII(
553 kIterationSwitch),
554 &g_iterations));
555 ASSERT_GT(g_iterations, 0);
558 if (parsed_command_line.HasSwitch(kMemoryUsageSwitch))
559 g_memory_usage = true;
561 if (parsed_command_line.HasSwitch(kLogFileSwitch))
562 g_log_file_path = parsed_command_line.GetSwitchValuePath(kLogFileSwitch);
564 if (parsed_command_line.HasSwitch(kTimeoutSwitch)) {
565 ASSERT_TRUE(
566 base::StringToInt(parsed_command_line.GetSwitchValueASCII(
567 kTimeoutSwitch),
568 &g_timeout_seconds));
569 ASSERT_GT(g_timeout_seconds, 0);
572 if (parsed_command_line.HasSwitch(kNoPageDownSwitch))
573 g_page_down = false;
575 if (parsed_command_line.HasSwitch(kNoClearProfileSwitch))
576 g_clear_profile = false;
578 if (parsed_command_line.HasSwitch(kSaveDebugLogSwitch)) {
579 g_save_debug_log = true;
580 g_chrome_log_path = logging::GetLogFileName();
581 // We won't get v8 log unless --no-sandbox is specified.
582 if (parsed_command_line.HasSwitch(switches::kNoSandbox)) {
583 PathService::Get(base::DIR_CURRENT, &g_v8_log_path);
584 g_v8_log_path = g_v8_log_path.AppendASCII(kV8LogFileDefaultName);
585 // The command line switch may override the default v8 log path.
586 if (parsed_command_line.HasSwitch(switches::kJavaScriptFlags)) {
587 CommandLine v8_command_line(
588 parsed_command_line.GetSwitchValuePath(switches::kJavaScriptFlags));
589 if (v8_command_line.HasSwitch(kV8LogFileSwitch)) {
590 g_v8_log_path = v8_command_line.GetSwitchValuePath(kV8LogFileSwitch);
591 if (!file_util::AbsolutePath(&g_v8_log_path))
592 g_v8_log_path = base::FilePath();