1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file provides reliablity tests which run for ChromeFrame.
8 // <reliability test exe> --list=file --startline=start --endline=end [...]
9 // Upon invocation, it visits each of the URLs on line numbers between start
10 // and end, inclusive, stored in the input file. The line number starts from 1.
13 // --iterations=num: goes through the list of URLs constructed in usage 2 or 3
15 // --memoryusage: prints out memory usage when visiting each page.
16 // --logfile=filepath: saves the visit log to the specified path.
17 // --timeout=seconds: time out as specified in seconds during each
19 // --nopagedown: won't simulate page down key presses after page load.
20 // --noclearprofile: do not clear profile dir before firing up each time.
21 // --savedebuglog: save Chrome, V8, and test debug log for each page loaded.
25 #include "base/command_line.h"
26 #include "base/file_util.h"
27 #include "base/file_version_info.h"
28 #include "base/files/file_path.h"
29 #include "base/i18n/time_formatting.h"
30 #include "base/path_service.h"
31 #include "base/prefs/json_pref_store.h"
32 #include "base/prefs/pref_registry_simple.h"
33 #include "base/prefs/pref_service.h"
34 #include "base/prefs/pref_value_store.h"
35 #include "base/string_number_conversions.h"
36 #include "base/string_util.h"
37 #include "base/test/test_file_util.h"
38 #include "base/time.h"
39 #include "base/utf_string_conversions.h"
40 #include "chrome/browser/net/url_fixer_upper.h"
41 #include "chrome/browser/prefs/pref_service_mock_builder.h"
42 #include "chrome/common/automation_messages.h"
43 #include "chrome/common/chrome_constants.h"
44 #include "chrome/common/chrome_paths.h"
45 #include "chrome/common/chrome_paths_internal.h"
46 #include "chrome/common/chrome_switches.h"
47 #include "chrome/common/logging_chrome.h"
48 #include "chrome/common/pref_names.h"
49 #include "chrome/test/automation/automation_proxy.h"
50 #include "chrome/test/automation/browser_proxy.h"
51 #include "chrome/test/automation/tab_proxy.h"
52 #include "chrome/test/automation/window_proxy.h"
53 #include "chrome/test/ui/ui_test.h"
54 #include "chrome_frame/test/chrome_frame_test_utils.h"
55 #include "chrome_frame/test/ie_event_sink.h"
56 #include "chrome_frame/test/reliability/page_load_test.h"
57 #include "chrome_frame/utils.h"
58 #include "content/public/browser/browser_thread.h"
59 #include "content/public/test/test_browser_thread.h"
60 #include "net/base/net_util.h"
61 #include "testing/gmock/include/gmock/gmock.h"
62 #include "testing/gtest/include/gtest/gtest.h"
64 using testing::StrCaseEq
;
65 using testing::StrCaseNe
;
69 // See comments at the beginning of the file for the definition of switches.
70 const char kListSwitch
[] = "list";
71 const char kStartIndexSwitch
[] = "startline";
72 const char kEndIndexSwitch
[] = "endline";
73 const char kIterationSwitch
[] = "iterations";
74 const char kContinuousLoadSwitch
[] = "continuousload";
75 const char kMemoryUsageSwitch
[] = "memoryusage";
76 const char kLogFileSwitch
[] = "logfile";
77 const char kTimeoutSwitch
[] = "timeout";
78 const char kNoPageDownSwitch
[] = "nopagedown";
79 const char kNoClearProfileSwitch
[] = "noclearprofile";
80 const char kSaveDebugLogSwitch
[] = "savedebuglog";
82 // These are copied from v8 definitions as we cannot include them.
83 const char kV8LogFileSwitch
[] = "logfile";
84 const char kV8LogFileDefaultName
[] = "v8.log";
86 // String name of local chrome dll for looking up file information.
87 const wchar_t kChromeDll
[] = L
"chrome.dll";
89 base::FilePath g_url_file_path
;
90 int32 g_start_index
= 1;
91 int32 g_end_index
= kint32max
;
92 int32 g_iterations
= 1;
93 bool g_memory_usage
= false;
94 bool g_page_down
= true;
95 bool g_clear_profile
= true;
96 std::string g_end_url
;
97 base::FilePath g_log_file_path
;
98 bool g_save_debug_log
= false;
99 base::FilePath g_chrome_log_path
;
100 base::FilePath g_v8_log_path
;
101 base::FilePath g_test_log_path
;
102 bool g_stand_alone
= false;
104 const int kUrlNavigationTimeoutSeconds
= 20;
105 int g_timeout_seconds
= kUrlNavigationTimeoutSeconds
;
107 // Mocks document complete and load events.
108 class MockLoadListener
: public chrome_frame_test::IEEventListener
{
110 MOCK_METHOD1(OnDocumentComplete
, void (const wchar_t* url
)); // NOLINT
111 MOCK_METHOD1(OnLoad
, void (const wchar_t* url
)); // NOLINT
112 MOCK_METHOD0(OnQuit
, void ()); // NOLINT
115 virtual void OnDocumentComplete(IDispatch
* dispatch
, VARIANT
* url
) {
117 OnDocumentComplete(url
->bstrVal
);
121 ACTION_P(QuitIE
, event_sink
) {
122 EXPECT_HRESULT_SUCCEEDED(event_sink
->CloseWebBrowser());
125 class PageLoadTest
: public testing::Test
{
127 enum NavigationResult
{
128 NAVIGATION_ERROR
= 0,
133 // These are results from the test automation that drives Chrome
134 NavigationResult result
;
135 int crash_dump_count
;
136 // These are stability metrics recorded by Chrome itself
137 bool browser_clean_exit
;
138 int browser_launch_count
;
140 int browser_crash_count
;
141 int renderer_crash_count
;
142 int plugin_crash_count
;
147 // Accept URL as std::string here because the url may also act as a test id
148 // and needs to be logged in its original format even if invalid.
149 void NavigateToURLLogResult(const std::string
& url_string
,
150 std::ofstream
& log_file
,
151 NavigationMetrics
* metrics_output
) {
152 GURL
url(url_string
);
153 NavigationMetrics metrics
= {NAVIGATION_ERROR
};
154 std::ofstream test_log
;
156 // Create a test log.
157 g_test_log_path
= base::FilePath(FILE_PATH_LITERAL("test_log.log"));
158 test_log
.open(g_test_log_path
.value().c_str());
160 // Check file version info for chrome dll.
161 scoped_ptr
<FileVersionInfo
> file_info
;
164 FileVersionInfo::CreateFileVersionInfo(base::FilePath(kChromeDll
)));
165 #elif defined(OS_LINUX) || defined(OS_MACOSX)
166 // TODO(fmeawad): the version retrieved here belongs to the test module and
167 // not the chrome binary, need to be changed to chrome binary instead.
168 file_info
.reset(FileVersionInfo::CreateFileVersionInfoForCurrentModule());
169 #endif // !defined(OS_WIN)
170 std::wstring last_change
= file_info
->last_change();
171 test_log
<< "Last Change: ";
172 test_log
<< last_change
<< std::endl
;
175 // Log timestamp for test start.
176 base::Time time_now
= base::Time::Now();
177 double time_start
= time_now
.ToDoubleT();
178 test_log
<< "Test Start: ";
179 test_log
<< base::TimeFormatFriendlyDateAndTime(time_now
) << std::endl
;
183 chrome_frame_test::TimedMsgLoop message_loop
;
186 base::win::ScopedComPtr
<IWebBrowser2
> web_browser2
;
187 hr
= chrome_frame_test::LaunchIEAsComServer(web_browser2
.Receive());
188 EXPECT_HRESULT_SUCCEEDED(hr
);
189 EXPECT_TRUE(web_browser2
.get() != NULL
);
190 web_browser2
->put_Visible(VARIANT_TRUE
);
192 // Log Browser Launched time.
193 time_now
= base::Time::Now();
194 test_log
<< "browser_launched_seconds=";
195 test_log
<< (time_now
.ToDoubleT() - time_start
) << std::endl
;
197 bool is_chrome_frame_navigation
=
198 StartsWith(UTF8ToWide(url
.spec()), kChromeProtocolPrefix
, true);
200 CComObjectStack
<chrome_frame_test::IEEventSink
> ie_event_sink
;
201 MockLoadListener load_listener
;
202 // Disregard any interstitial about:blank loads.
203 EXPECT_CALL(load_listener
, OnDocumentComplete(StrCaseEq(L
"about:blank")))
204 .Times(testing::AnyNumber());
206 // Note that we can't compare the loaded url directly with the given url
207 // because the page may have redirected us to a different page, e.g.
208 // www.google.com -> www.google.ca.
209 if (is_chrome_frame_navigation
) {
210 EXPECT_CALL(load_listener
, OnDocumentComplete(testing::_
));
211 EXPECT_CALL(load_listener
, OnLoad(testing::_
))
212 .WillOnce(QuitIE(&ie_event_sink
));
214 EXPECT_CALL(load_listener
, OnDocumentComplete(StrCaseNe(L
"about:blank")))
215 .WillOnce(QuitIE(&ie_event_sink
));
217 EXPECT_CALL(load_listener
, OnQuit()).WillOnce(QUIT_LOOP(message_loop
));
219 // Attach the sink and navigate.
220 ie_event_sink
.set_listener(&load_listener
);
221 ie_event_sink
.Attach(web_browser2
);
222 hr
= ie_event_sink
.Navigate(UTF8ToWide(url
.spec()));
224 message_loop
.RunFor(base::TimeDelta::FromSeconds(g_timeout_seconds
));
225 if (!message_loop
.WasTimedOut())
226 metrics
.result
= NAVIGATION_SUCCESS
;
229 // Log navigate complete time.
230 time_now
= base::Time::Now();
231 test_log
<< "navigate_complete_seconds=";
232 test_log
<< (time_now
.ToDoubleT() - time_start
) << std::endl
;
235 ie_event_sink
.set_listener(NULL
);
236 ie_event_sink
.Uninitialize();
237 chrome_frame_test::CloseAllIEWindows();
239 // Log end of test time.
240 time_now
= base::Time::Now();
241 test_log
<< "total_duration_seconds=";
242 test_log
<< (time_now
.ToDoubleT() - time_start
) << std::endl
;
244 // Get navigation result and metrics, and optionally write to the log file
245 // provided. The log format is:
246 // <url> <navigation_result> <browser_crash_count> <renderer_crash_count>
247 // <plugin_crash_count> <crash_dump_count> [chrome_log=<path>
248 // v8_log=<path>] crash_dump=<path>
249 if (log_file
.is_open()) {
250 log_file
<< url_string
;
251 switch (metrics
.result
) {
252 case NAVIGATION_ERROR
:
253 log_file
<< " error";
255 case NAVIGATION_SUCCESS
:
256 log_file
<< " success";
263 // Get stability metrics recorded by Chrome itself.
264 if (is_chrome_frame_navigation
) {
265 GetStabilityMetrics(&metrics
);
268 if (log_file
.is_open()) {
269 log_file
<< " " << metrics
.browser_crash_count \
270 // The renderer crash count is flaky due to 1183283.
271 // Ignore the count since we also catch crash by
274 << " " << metrics
.plugin_crash_count \
275 << " " << metrics
.crash_dump_count
;
281 if (log_file
.is_open() && g_save_debug_log
)
282 SaveDebugLogs(log_file
);
284 // Log revision information for Chrome build under test.
285 log_file
<< " " << "revision=" << last_change
;
288 LogOrDeleteNewCrashDumps(log_file
, &metrics
);
290 if (log_file
.is_open()) {
291 log_file
<< std::endl
;
294 if (metrics_output
) {
295 *metrics_output
= metrics
;
299 void NavigateThroughURLList(std::ofstream
& log_file
) {
300 std::ifstream
file(g_url_file_path
.value().c_str());
301 ASSERT_TRUE(file
.is_open());
303 // We navigate to URLs in the following order.
304 // CF -> CF -> host -> CF -> CF -> host.
305 for (int line_index
= 1;
306 line_index
<= g_end_index
&& !file
.eof();
309 std::getline(file
, url_str
);
315 // Every 3rd URL goes into the host browser.
316 if (line_index
% 3 != 0) {
317 std::string actual_url
;
318 actual_url
= WideToUTF8(kChromeProtocolPrefix
);
319 actual_url
+= url_str
;
320 url_str
= actual_url
;
323 if (g_start_index
<= line_index
) {
324 NavigateToURLLogResult(url_str
, log_file
, NULL
);
332 virtual void SetUp() {
333 // Initialize crash_dumps_dir_path_.
334 PathService::Get(chrome::DIR_CRASH_DUMPS
, &crash_dumps_dir_path_
);
335 file_util::FileEnumerator
enumerator(crash_dumps_dir_path_
,
336 false, // not recursive
337 file_util::FileEnumerator::FILES
);
338 for (base::FilePath path
= enumerator
.Next(); !path
.value().empty();
339 path
= enumerator
.Next()) {
340 if (path
.MatchesExtension(FILE_PATH_LITERAL(".dmp")))
341 crash_dumps_
[path
.BaseName()] = true;
344 if (g_clear_profile
) {
345 base::FilePath user_data_dir
;
346 chrome::GetChromeFrameUserDataDirectory(&user_data_dir
);
347 ASSERT_TRUE(file_util::DieFileDie(user_data_dir
, true));
350 SetConfigBool(kChromeFrameHeadlessMode
, true);
351 SetConfigBool(kAllowUnsafeURLs
, true);
354 virtual void TearDown() {
355 DeleteConfigValue(kChromeFrameHeadlessMode
);
356 DeleteConfigValue(kAllowUnsafeURLs
);
359 base::FilePath
ConstructSavedDebugLogPath(const base::FilePath
& debug_log_path
,
361 std::string
suffix("_");
362 suffix
.append(base::IntToString(index
));
363 return debug_log_path
.InsertBeforeExtensionASCII(suffix
);
366 void SaveDebugLog(const base::FilePath
& log_path
, const std::wstring
& log_id
,
367 std::ofstream
& log_file
, int index
) {
368 if (!log_path
.empty()) {
369 base::FilePath saved_log_file_path
=
370 ConstructSavedDebugLogPath(log_path
, index
);
371 if (file_util::Move(log_path
, saved_log_file_path
)) {
372 log_file
<< " " << log_id
<< "=" << saved_log_file_path
.value();
377 // Rename the chrome and v8 debug log files if existing, and save the file
378 // paths in the log_file provided.
379 void SaveDebugLogs(std::ofstream
& log_file
) {
380 static int url_count
= 1;
381 SaveDebugLog(g_chrome_log_path
, L
"chrome_log", log_file
, url_count
);
382 SaveDebugLog(g_v8_log_path
, L
"v8_log", log_file
, url_count
);
383 SaveDebugLog(g_test_log_path
, L
"test_log", log_file
, url_count
);
387 // If a log_file is provided, log the crash dump with the given path;
388 // otherwise, delete the crash dump file.
389 void LogOrDeleteCrashDump(std::ofstream
& log_file
,
390 base::FilePath crash_dump_file_name
) {
391 base::FilePath
crash_dump_file_path(crash_dumps_dir_path_
);
392 crash_dump_file_path
= crash_dump_file_path
.Append(crash_dump_file_name
);
393 base::FilePath crash_text_file_path
=
394 crash_dump_file_path
.ReplaceExtension(FILE_PATH_LITERAL("txt"));
396 if (log_file
.is_open()) {
397 crash_dumps_
[crash_dump_file_name
] = true;
398 log_file
<< " crash_dump=" << crash_dump_file_path
.value().c_str();
400 ASSERT_TRUE(file_util::DieFileDie(
401 crash_dump_file_path
, false));
402 ASSERT_TRUE(file_util::DieFileDie(
403 crash_text_file_path
, false));
407 // Check whether there are new .dmp files. Additionally, write
408 // " crash_dump=<full path name of the .dmp file>"
410 void LogOrDeleteNewCrashDumps(std::ofstream
& log_file
,
411 NavigationMetrics
* metrics
) {
414 file_util::FileEnumerator
enumerator(crash_dumps_dir_path_
,
415 false, // not recursive
416 file_util::FileEnumerator::FILES
);
417 for (base::FilePath path
= enumerator
.Next(); !path
.value().empty();
418 path
= enumerator
.Next()) {
419 if (path
.MatchesExtension(FILE_PATH_LITERAL(".dmp")) &&
420 !crash_dumps_
[path
.BaseName()]) {
421 LogOrDeleteCrashDump(log_file
, path
.BaseName());
426 metrics
->crash_dump_count
= num_dumps
;
429 // Get a PrefService whose contents correspond to the Local State file
430 // that was saved by the app as it closed. The caller takes ownership of the
431 // returned PrefService object.
432 PrefService
* GetLocalState(PrefRegistry
* registry
) {
434 chrome::GetChromeFrameUserDataDirectory(&path
);
435 PrefServiceMockBuilder builder
;
436 builder
.WithUserFilePrefs(
438 JsonPrefStore::GetTaskRunnerForFile(
439 path
, content::BrowserThread::GetBlockingPool()));
440 return builder
.Create(registry
);
443 void GetStabilityMetrics(NavigationMetrics
* metrics
) {
446 scoped_refptr
<PrefRegistrySimple
> registry
= new PrefRegistrySimple();
447 registry
->RegisterBooleanPref(prefs::kStabilityExitedCleanly
, false);
448 registry
->RegisterIntegerPref(prefs::kStabilityLaunchCount
, -1);
449 registry
->RegisterIntegerPref(prefs::kStabilityPageLoadCount
, -1);
450 registry
->RegisterIntegerPref(prefs::kStabilityCrashCount
, 0);
451 registry
->RegisterIntegerPref(prefs::kStabilityRendererCrashCount
, 0);
453 scoped_ptr
<PrefService
> local_state(GetLocalState(registry
));
454 if (!local_state
.get())
457 metrics
->browser_clean_exit
=
458 local_state
->GetBoolean(prefs::kStabilityExitedCleanly
);
459 metrics
->browser_launch_count
=
460 local_state
->GetInteger(prefs::kStabilityLaunchCount
);
461 metrics
->page_load_count
=
462 local_state
->GetInteger(prefs::kStabilityPageLoadCount
);
463 metrics
->browser_crash_count
=
464 local_state
->GetInteger(prefs::kStabilityCrashCount
);
465 metrics
->renderer_crash_count
=
466 local_state
->GetInteger(prefs::kStabilityRendererCrashCount
);
468 metrics
->plugin_crash_count
= 0;
470 if (!metrics
->browser_clean_exit
)
471 metrics
->browser_crash_count
++;
474 base::FilePath
GetSampleDataDir() {
475 base::FilePath test_dir
;
476 PathService::Get(chrome::DIR_TEST_DATA
, &test_dir
);
477 test_dir
= test_dir
.AppendASCII("reliability");
478 test_dir
= test_dir
.AppendASCII("sample_pages");
482 // The pathname of Chrome's crash dumps directory.
483 base::FilePath crash_dumps_dir_path_
;
485 // The set of all the crash dumps we have seen. Each crash generates a
486 // .dmp and a .txt file in the crash dumps directory. We only store the
487 // .dmp files in this set.
489 // The set is implemented as a std::map. The key is the file name, and
490 // the value is false (the file is not in the set) or true (the file is
491 // in the set). The initial value for any key in std::map is 0 (false),
492 // which in this case means a new file is not in the set initially,
493 // exactly the semantics we want.
494 std::map
<base::FilePath
, bool> crash_dumps_
;
497 TEST_F(PageLoadTest
, IEFullTabMode_Reliability
) {
498 std::ofstream log_file
;
500 if (!g_log_file_path
.empty()) {
501 log_file
.open(g_log_file_path
.value().c_str());
504 EXPECT_FALSE(g_url_file_path
.empty());
506 for (int k
= 0; k
< g_iterations
; ++k
) {
507 NavigateThroughURLList(log_file
);
516 void ReportHandler(const std::string
& str
) {
517 // Ignore report events.
521 void SetPageRange(const CommandLine
& parsed_command_line
) {
522 // If calling into this function, we are running as a standalone program.
523 g_stand_alone
= true;
525 // Since we use --enable-dcheck for reliability tests, suppress the error
526 // dialog in the test process.
527 logging::SetLogReportHandler(ReportHandler
);
529 if (parsed_command_line
.HasSwitch(kStartIndexSwitch
)) {
531 base::StringToInt(parsed_command_line
.GetSwitchValueASCII(
534 ASSERT_GT(g_start_index
, 0);
537 if (parsed_command_line
.HasSwitch(kEndIndexSwitch
)) {
539 base::StringToInt(parsed_command_line
.GetSwitchValueASCII(
542 ASSERT_GT(g_end_index
, 0);
545 ASSERT_TRUE(g_end_index
>= g_start_index
);
547 if (parsed_command_line
.HasSwitch(kListSwitch
))
548 g_url_file_path
= parsed_command_line
.GetSwitchValuePath(kListSwitch
);
550 if (parsed_command_line
.HasSwitch(kIterationSwitch
)) {
552 base::StringToInt(parsed_command_line
.GetSwitchValueASCII(
555 ASSERT_GT(g_iterations
, 0);
558 if (parsed_command_line
.HasSwitch(kMemoryUsageSwitch
))
559 g_memory_usage
= true;
561 if (parsed_command_line
.HasSwitch(kLogFileSwitch
))
562 g_log_file_path
= parsed_command_line
.GetSwitchValuePath(kLogFileSwitch
);
564 if (parsed_command_line
.HasSwitch(kTimeoutSwitch
)) {
566 base::StringToInt(parsed_command_line
.GetSwitchValueASCII(
568 &g_timeout_seconds
));
569 ASSERT_GT(g_timeout_seconds
, 0);
572 if (parsed_command_line
.HasSwitch(kNoPageDownSwitch
))
575 if (parsed_command_line
.HasSwitch(kNoClearProfileSwitch
))
576 g_clear_profile
= false;
578 if (parsed_command_line
.HasSwitch(kSaveDebugLogSwitch
)) {
579 g_save_debug_log
= true;
580 g_chrome_log_path
= logging::GetLogFileName();
581 // We won't get v8 log unless --no-sandbox is specified.
582 if (parsed_command_line
.HasSwitch(switches::kNoSandbox
)) {
583 PathService::Get(base::DIR_CURRENT
, &g_v8_log_path
);
584 g_v8_log_path
= g_v8_log_path
.AppendASCII(kV8LogFileDefaultName
);
585 // The command line switch may override the default v8 log path.
586 if (parsed_command_line
.HasSwitch(switches::kJavaScriptFlags
)) {
587 CommandLine
v8_command_line(
588 parsed_command_line
.GetSwitchValuePath(switches::kJavaScriptFlags
));
589 if (v8_command_line
.HasSwitch(kV8LogFileSwitch
)) {
590 g_v8_log_path
= v8_command_line
.GetSwitchValuePath(kV8LogFileSwitch
);
591 if (!file_util::AbsolutePath(&g_v8_log_path
))
592 g_v8_log_path
= base::FilePath();