chrome/test/perf/startup_test.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/environment.h"
   6 #include "base/file_util.h"
   7 #include "base/files/file_enumerator.h"
   8 #include "base/path_service.h"
   9 #include "base/strings/string_number_conversions.h"
  10 #include "base/strings/string_split.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "base/strings/utf_string_conversions.h"
  14 #include "base/sys_info.h"
  15 #include "base/test/test_file_util.h"
  16 #include "base/test/test_timeouts.h"
  17 #include "base/time/time.h"
  18 #include "chrome/common/chrome_constants.h"
  19 #include "chrome/common/chrome_paths.h"
  20 #include "chrome/common/chrome_switches.h"
  21 #include "chrome/common/env_vars.h"
  22 #include "chrome/test/automation/automation_proxy.h"
  23 #include "chrome/test/base/test_switches.h"
  24 #include "chrome/test/base/testing_profile.h"
  25 #include "chrome/test/base/ui_test_utils.h"
  26 #include "chrome/test/perf/perf_test.h"
  27 #include "chrome/test/perf/perf_ui_test_suite.h"
  28 #include "chrome/test/ui/ui_perf_test.h"
  29 #include "content/public/common/content_switches.h"
  30 #include "net/base/net_util.h"
  31 #include "testing/perf/perf_test.h"
  32
  33 using base::TimeDelta;
  34 using base::TimeTicks;
  35
  36 namespace {
  37
  38 class StartupTest : public UIPerfTest {
  39  public:
  40   StartupTest() {
  41     show_window_ = true;
  42   }
  43   virtual void SetUp() {
  44     collect_profiling_stats_ = false;
  45   }
  46   virtual void TearDown() {}
  47
  48   enum TestColdness {
  49     WARM,
  50     COLD
  51   };
  52
  53   enum TestImportance {
  54     NOT_IMPORTANT,
  55     IMPORTANT
  56   };
  57
  58   // Load a file on startup rather than about:blank.  This tests a longer
  59   // startup path, including resource loading.
  60   void SetUpWithFileURL() {
  61     const base::FilePath file_url = ui_test_utils::GetTestFilePath(
  62         base::FilePath(base::FilePath::kCurrentDirectory),
  63         base::FilePath(FILE_PATH_LITERAL("simple.html")));
  64     ASSERT_TRUE(base::PathExists(file_url));
  65     launch_arguments_.AppendArgPath(file_url);
  66   }
  67
  68   // Setup the command line arguments to capture profiling data for tasks.
  69   void SetUpWithProfiling() {
  70     profiling_file_ = ui_test_utils::GetTestFilePath(
  71         base::FilePath(base::FilePath::kCurrentDirectory),
  72         base::FilePath(FILE_PATH_LITERAL("task_profile.json")));
  73     launch_arguments_.AppendSwitchPath(switches::kProfilingOutputFile,
  74                                        profiling_file_);
  75     collect_profiling_stats_ = true;
  76   }
  77
  78   // Load a complex html file on startup represented by |which_tab|.
  79   void SetUpWithComplexFileURL(unsigned int which_tab) {
  80     const char* const kTestPageCyclerDomains[] = {
  81         "www.google.com", "www.nytimes.com",
  82         "www.yahoo.com", "espn.go.com", "www.amazon.com"
  83     };
  84     unsigned int which_el = which_tab % arraysize(kTestPageCyclerDomains);
  85     const char* this_domain = kTestPageCyclerDomains[which_el];
  86
  87     base::FilePath page_cycler_path;
  88     PathService::Get(base::DIR_SOURCE_ROOT, &page_cycler_path);
  89     page_cycler_path = page_cycler_path.AppendASCII("data")
  90         .AppendASCII("page_cycler").AppendASCII("moz")
  91         .AppendASCII(this_domain).AppendASCII("index.html");
  92     GURL file_url = net::FilePathToFileURL(page_cycler_path).Resolve("?skip");
  93     launch_arguments_.AppendArg(file_url.spec());
  94   }
  95
  96   // Use the given profile in the test data extensions/profiles dir.  This tests
  97   // startup with extensions installed.
  98   void SetUpWithExtensionsProfile(const char* profile) {
  99     base::FilePath data_dir;
 100     PathService::Get(chrome::DIR_TEST_DATA, &data_dir);
 101     data_dir = data_dir.AppendASCII("extensions").AppendASCII("profiles").
 102         AppendASCII(profile);
 103     set_template_user_data(data_dir);
 104   }
 105
 106   // Rewrite the preferences file to point to the proper image directory.
 107   virtual void SetUpProfile() OVERRIDE {
 108     UIPerfTest::SetUpProfile();
 109     if (profile_type_ != PerfUITestSuite::COMPLEX_THEME)
 110       return;
 111
 112     const base::FilePath pref_template_path(user_data_dir().
 113         AppendASCII("Default").
 114         AppendASCII("PreferencesTemplate"));
 115     const base::FilePath pref_path(user_data_dir().
 116         AppendASCII(TestingProfile::kTestUserProfileDir).
 117         AppendASCII("Preferences"));
 118
 119     // Read in preferences template.
 120     std::string pref_string;
 121     EXPECT_TRUE(base::ReadFileToString(pref_template_path, &pref_string));
 122     string16 format_string = ASCIIToUTF16(pref_string);
 123
 124     // Make sure temp directory has the proper format for writing to prefs file.
 125 #if defined(OS_POSIX)
 126     std::wstring user_data_dir_w(ASCIIToWide(user_data_dir().value()));
 127 #elif defined(OS_WIN)
 128     std::wstring user_data_dir_w(user_data_dir().value());
 129     // In Windows, the FilePath will write '\' for the path separators; change
 130     // these to a separator that won't trigger escapes.
 131     std::replace(user_data_dir_w.begin(),
 132                  user_data_dir_w.end(), '\\', '/');
 133 #endif
 134
 135     // Rewrite prefs file.
 136     std::vector<string16> subst;
 137     subst.push_back(WideToUTF16(user_data_dir_w));
 138     const std::string prefs_string =
 139         UTF16ToASCII(ReplaceStringPlaceholders(format_string, subst, NULL));
 140     EXPECT_TRUE(file_util::WriteFile(pref_path, prefs_string.c_str(),
 141                                      prefs_string.size()));
 142     file_util::EvictFileFromSystemCache(pref_path);
 143   }
 144
 145   // Runs a test which loads |tab_count| tabs on startup, either as command line
 146   // arguments or, if |restore_session| is true, by using session restore.
 147   // |nth_timed_tab|, if non-zero, will measure time to load the first n+1 tabs.
 148   void RunPerfTestWithManyTabs(const char* graph, const char* trace,
 149                                int tab_count, int nth_timed_tab,
 150                                bool restore_session);
 151
 152   void RunStartupTest(const char* graph, const char* trace,
 153                       TestColdness test_cold, TestImportance test_importance,
 154                       PerfUITestSuite::ProfileType profile_type,
 155                       int num_tabs, int nth_timed_tab) {
 156     bool important = (test_importance == IMPORTANT);
 157     profile_type_ = profile_type;
 158
 159     // Sets the profile data for the run.  For now, this is only used for
 160     // the non-default themes test.
 161     if (profile_type != PerfUITestSuite::DEFAULT_THEME) {
 162       set_template_user_data(
 163           PerfUITestSuite::GetPathForProfileType(profile_type));
 164     }
 165
 166 #if defined(NDEBUG)
 167     const int kNumCyclesMax = 20;
 168 #else
 169     // Debug builds are too slow and we can't run that many cycles in a
 170     // reasonable amount of time.
 171     const int kNumCyclesMax = 10;
 172 #endif
 173     int numCycles = kNumCyclesMax;
 174     scoped_ptr<base::Environment> env(base::Environment::Create());
 175     std::string numCyclesEnv;
 176     if (env->GetVar(env_vars::kStartupTestsNumCycles, &numCyclesEnv) &&
 177         base::StringToInt(numCyclesEnv, &numCycles)) {
 178       if (numCycles <= kNumCyclesMax) {
 179         VLOG(1) << env_vars::kStartupTestsNumCycles
 180                 << " set in environment, so setting numCycles to " << numCycles;
 181       } else {
 182         VLOG(1) << env_vars::kStartupTestsNumCycles
 183                 << " is higher than the max, setting numCycles to "
 184                 << kNumCyclesMax;
 185         numCycles = kNumCyclesMax;
 186       }
 187     }
 188
 189     struct TimingInfo {
 190       TimeDelta end_to_end;
 191       float first_start_ms;
 192       float last_stop_ms;
 193       float first_stop_ms;
 194       float nth_tab_stop_ms;
 195     };
 196     TimingInfo timings[kNumCyclesMax];
 197
 198     CommandLine launch_arguments_without_trace_file(launch_arguments_);
 199     for (int i = 0; i < numCycles; ++i) {
 200       if (test_cold == COLD) {
 201         base::FilePath dir_app;
 202         ASSERT_TRUE(PathService::Get(chrome::DIR_APP, &dir_app));
 203
 204         base::FilePath chrome_exe(dir_app.Append(GetExecutablePath()));
 205         ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_exe));
 206 #if defined(OS_WIN)
 207         // chrome.dll is windows specific.
 208         base::FilePath chrome_dll(
 209             dir_app.Append(FILE_PATH_LITERAL("chrome.dll")));
 210         ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_dll));
 211 #endif
 212       }
 213       UITest::SetUp();
 214       TimeTicks end_time = TimeTicks::Now();
 215
 216       if (num_tabs > 0) {
 217         float min_start;
 218         float max_stop;
 219         std::vector<float> times;
 220         scoped_refptr<BrowserProxy> browser_proxy(
 221             automation()->GetBrowserWindow(0));
 222         ASSERT_TRUE(browser_proxy.get());
 223
 224         if (browser_proxy->GetInitialLoadTimes(
 225               TestTimeouts::action_max_timeout(),
 226               &min_start,
 227               &max_stop,
 228               &times) &&
 229             !times.empty()) {
 230           ASSERT_LT(nth_timed_tab, num_tabs);
 231           ASSERT_EQ(times.size(), static_cast<size_t>(num_tabs));
 232           timings[i].first_start_ms = min_start;
 233           timings[i].last_stop_ms = max_stop;
 234           timings[i].first_stop_ms = times[0];
 235           timings[i].nth_tab_stop_ms = times[nth_timed_tab];
 236         } else {
 237           // Browser might not support initial load times.
 238           // Only use end-to-end time for this test.
 239           num_tabs = 0;
 240         }
 241       }
 242       timings[i].end_to_end = end_time - browser_launch_time();
 243       UITest::TearDown();
 244
 245       if (i == 0) {
 246         // Re-use the profile data after first run so that the noise from
 247         // creating databases doesn't impact all the runs.
 248         clear_profile_ = false;
 249         // Clear template_user_data_ so we don't try to copy it over each time
 250         // through.
 251         set_template_user_data(base::FilePath());
 252       }
 253     }
 254
 255     std::string times;
 256     for (int i = 0; i < numCycles; ++i) {
 257       base::StringAppendF(&times,
 258                           "%.2f,",
 259                           timings[i].end_to_end.InMillisecondsF());
 260     }
 261     perf_test::PrintResultList(
 262         graph, std::string(), trace, times, "ms", important);
 263
 264     if (num_tabs > 0) {
 265       std::string name_base = trace;
 266       std::string name;
 267
 268       times.clear();
 269       name = name_base + "-start";
 270       for (int i = 0; i < numCycles; ++i)
 271         base::StringAppendF(&times, "%.2f,", timings[i].first_start_ms);
 272       perf_test::PrintResultList(
 273           graph, std::string(), name.c_str(), times, "ms", important);
 274
 275       times.clear();
 276       name = name_base + "-first";
 277       for (int i = 0; i < numCycles; ++i)
 278         base::StringAppendF(&times, "%.2f,", timings[i].first_stop_ms);
 279       perf_test::PrintResultList(
 280           graph, std::string(), name.c_str(), times, "ms", important);
 281
 282       if (nth_timed_tab > 0) {
 283         // Display only the time necessary to load the first n tabs.
 284         times.clear();
 285         name = name_base + "-" + base::IntToString(nth_timed_tab);
 286         for (int i = 0; i < numCycles; ++i)
 287           base::StringAppendF(&times, "%.2f,", timings[i].nth_tab_stop_ms);
 288         perf_test::PrintResultList(
 289             graph, std::string(), name.c_str(), times, "ms", important);
 290       }
 291
 292       if (num_tabs > 1) {
 293         // Display the time necessary to load all of the tabs.
 294         times.clear();
 295         name = name_base + "-all";
 296         for (int i = 0; i < numCycles; ++i)
 297           base::StringAppendF(&times, "%.2f,", timings[i].last_stop_ms);
 298         perf_test::PrintResultList(
 299             graph, std::string(), name.c_str(), times, "ms", important);
 300       }
 301     }
 302   }
 303
 304   base::FilePath profiling_file_;
 305   bool collect_profiling_stats_;
 306   std::string trace_file_prefix_;
 307
 308   // Are we using a profile with a complex theme?
 309   PerfUITestSuite::ProfileType profile_type_;
 310 };
 311
 312 TEST_F(StartupTest, PerfWarm) {
 313   RunStartupTest("warm", "t", WARM, IMPORTANT,
 314                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
 315 }
 316
 317 TEST_F(StartupTest, PerfReferenceWarm) {
 318   UseReferenceBuild();
 319   RunStartupTest("warm", "t_ref", WARM, IMPORTANT,
 320                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
 321 }
 322
 323 // TODO(mpcomplete): Should we have reference timings for all these?
 324
 325 // dominich: Disabling as per http://crbug.com/100900.
 326 #if defined(OS_WIN)
 327 #define MAYBE_PerfCold DISABLED_PerfCold
 328 #else
 329 #define MAYBE_PerfCold PerfCold
 330 #endif
 331
 332 TEST_F(StartupTest, MAYBE_PerfCold) {
 333   RunStartupTest("cold", "t", COLD, NOT_IMPORTANT,
 334                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
 335 }
 336
 337 void StartupTest::RunPerfTestWithManyTabs(const char* graph, const char* trace,
 338                                           int tab_count, int nth_timed_tab,
 339                                           bool restore_session) {
 340   // Initialize session with |tab_count| tabs.
 341   for (int i = 0; i < tab_count; ++i)
 342     SetUpWithComplexFileURL(i);
 343
 344   if (restore_session) {
 345     // Start the browser with these urls so we can save the session and exit.
 346     UITest::SetUp();
 347     // Set flags to ensure profile is saved and can be restored.
 348 #if defined(OS_CHROMEOS) || defined(OS_MACOSX)
 349     set_shutdown_type(ProxyLauncher::USER_QUIT);
 350 #endif
 351     clear_profile_ = false;
 352     // Quit and set flags to restore session.
 353     UITest::TearDown();
 354
 355     // Clear all arguments for session restore, or the number of open tabs
 356     // will grow with each restore.
 357     CommandLine new_launch_arguments = CommandLine(CommandLine::NO_PROGRAM);
 358     // Keep the branding switch if using a reference build.
 359     if (launch_arguments_.HasSwitch(switches::kEnableChromiumBranding)) {
 360       new_launch_arguments.AppendSwitch(switches::kEnableChromiumBranding);
 361     }
 362     // The session will be restored once per cycle for numCycles test cycles,
 363     // and each time, UITest::SetUp will wait for |tab_count| tabs to
 364     // finish loading.
 365     new_launch_arguments.AppendSwitchASCII(switches::kRestoreLastSession,
 366                                         base::IntToString(tab_count));
 367     launch_arguments_ = new_launch_arguments;
 368   }
 369   RunStartupTest(graph, trace, WARM, NOT_IMPORTANT,
 370                  PerfUITestSuite::DEFAULT_THEME, tab_count, nth_timed_tab);
 371 }
 372
 373 // http://crbug.com/101591
 374 #if defined(OS_WIN) && !defined(NDEBUG)
 375 #define MAYBE_PerfFewTabs DISABLED_PerfFewTabs
 376 #else
 377 #define MAYBE_PerfFewTabs PerfFewTabs
 378 #endif
 379
 380 TEST_F(StartupTest, MAYBE_PerfFewTabs) {
 381   RunPerfTestWithManyTabs("few_tabs", "cmdline", 5, 2, false);
 382 }
 383
 384 TEST_F(StartupTest, PerfFewTabsReference) {
 385   UseReferenceBuild();
 386   RunPerfTestWithManyTabs("few_tabs", "cmdline-ref", 5, 2, false);
 387 }
 388
 389 TEST_F(StartupTest, PerfRestoreFewTabs) {
 390   RunPerfTestWithManyTabs("few_tabs", "restore", 5, 2, true);
 391 }
 392
 393 TEST_F(StartupTest, PerfRestoreFewTabsReference) {
 394   UseReferenceBuild();
 395   RunPerfTestWithManyTabs("few_tabs", "restore-ref", 5, 2, true);
 396 }
 397
 398 #if defined(OS_MACOSX)
 399 // http://crbug.com/46609
 400 #define MAYBE_PerfSeveralTabsReference DISABLED_PerfSeveralTabsReference
 401 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
 402 // http://crbug.com/52858
 403 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
 404 #define MAYBE_PerfExtensionContentScript50 DISABLED_PerfExtensionContentScript50
 405 #elif defined(OS_WIN)
 406 // http://crbug.com/46609
 407 #if !defined(NDEBUG)
 408 // This test is disabled on Windows Debug. See bug http://crbug.com/132279
 409 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
 410 #else
 411 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
 412 #endif  // !defined(NDEBUG)
 413 // http://crbug.com/102584
 414 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
 415 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
 416 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
 417 #else
 418 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
 419 #define MAYBE_PerfSeveralTabs PerfSeveralTabs
 420 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
 421 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
 422 #endif
 423
 424 // http://crbug.com/99604
 425 #if defined(OS_WIN) && !defined(NDEBUG)
 426 #define MAYBE_PerfComplexTheme DISABLED_PerfComplexTheme
 427 #else
 428 #define MAYBE_PerfComplexTheme PerfComplexTheme
 429 #endif
 430
 431 TEST_F(StartupTest, MAYBE_PerfSeveralTabs) {
 432   RunPerfTestWithManyTabs("several_tabs", "cmdline", 10, 4, false);
 433 }
 434
 435 TEST_F(StartupTest, MAYBE_PerfSeveralTabsReference) {
 436   UseReferenceBuild();
 437   RunPerfTestWithManyTabs("several_tabs", "cmdline-ref", 10, 4, false);
 438 }
 439
 440 TEST_F(StartupTest, MAYBE_PerfRestoreSeveralTabs) {
 441   RunPerfTestWithManyTabs("several_tabs", "restore", 10, 4, true);
 442 }
 443
 444 TEST_F(StartupTest, PerfRestoreSeveralTabsReference) {
 445   UseReferenceBuild();
 446   RunPerfTestWithManyTabs("several_tabs", "restore-ref", 10, 4, true);
 447 }
 448
 449 TEST_F(StartupTest, PerfExtensionEmpty) {
 450   SetUpWithFileURL();
 451   SetUpWithExtensionsProfile("empty");
 452   RunStartupTest("warm", "extension_empty", WARM, NOT_IMPORTANT,
 453                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
 454 }
 455
 456 TEST_F(StartupTest, PerfExtensionContentScript1) {
 457   SetUpWithFileURL();
 458   SetUpWithExtensionsProfile("content_scripts1");
 459   RunStartupTest("warm", "extension_content_scripts1", WARM, NOT_IMPORTANT,
 460                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
 461 }
 462
 463 TEST_F(StartupTest, MAYBE_PerfExtensionContentScript50) {
 464   SetUpWithFileURL();
 465   SetUpWithExtensionsProfile("content_scripts50");
 466   RunStartupTest("warm", "extension_content_scripts50", WARM, NOT_IMPORTANT,
 467                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
 468 }
 469
 470 TEST_F(StartupTest, MAYBE_PerfComplexTheme) {
 471   RunStartupTest("warm", "t-theme", WARM, NOT_IMPORTANT,
 472                  PerfUITestSuite::COMPLEX_THEME, 0, 0);
 473 }
 474
 475 TEST_F(StartupTest, ProfilingScript1) {
 476   SetUpWithFileURL();
 477   SetUpWithProfiling();
 478   RunStartupTest("warm", "profiling_scripts1", WARM, NOT_IMPORTANT,
 479                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
 480 }
 481
 482 }  // namespace