tools/perf/profile_creators/fast_navigation_profile_extender.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import time
   5
   6 from profile_creators import profile_extender
   7 from telemetry.core import browser_finder
   8 from telemetry.core import browser_finder_exceptions
   9 from telemetry.core import exceptions
  10 from telemetry.core import platform
  11
  12
  13 class FastNavigationProfileExtender(profile_extender.ProfileExtender):
  14   """Extends a Chrome profile.
  15
  16   This class creates or extends an existing profile by performing a set of tab
  17   navigations in large batches. This is accomplished by opening a large number
  18   of tabs, simultaneously navigating all the tabs, and then waiting for all the
  19   tabs to load. This provides two benefits:
  20     - Takes advantage of the high number of logical cores on modern CPUs.
  21     - The total time spent waiting for navigations to time out scales linearly
  22       with the number of batches, but does not scale with the size of the
  23       batch.
  24   """
  25   def __init__(self, maximum_batch_size):
  26     """Initializer.
  27
  28     Args:
  29       maximum_batch_size: A positive integer indicating the number of tabs to
  30       simultaneously perform navigations.
  31     """
  32     super(FastNavigationProfileExtender, self).__init__()
  33
  34     # The path of the profile that the browser will use while it's running.
  35     # This member is initialized during SetUp().
  36     self._profile_path = None
  37
  38     # A reference to the browser that will be performing all of the tab
  39     # navigations.
  40     # This member is initialized during SetUp().
  41     self._browser = None
  42
  43     # The instance keeps a list of Tabs that can be navigated successfully.
  44     # This means that the Tab is not crashed, and is processing JavaScript in a
  45     # timely fashion.
  46     self._navigation_tabs = []
  47
  48     # The number of tabs to use.
  49     self._NUM_TABS = maximum_batch_size
  50
  51     # The amount of time to wait for a batch of pages to finish loading.
  52     self._BATCH_PAGE_LOAD_TIMEOUT_IN_SECONDS = 10
  53
  54     # The default amount of time to wait for the retrieval of the URL of a tab.
  55     self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS = 1
  56
  57   def Run(self, finder_options):
  58     """Extends the profile.
  59
  60     Args:
  61       finder_options: An instance of BrowserFinderOptions that contains the
  62       directory of the input profile, the directory to place the output
  63       profile, and sufficient information to choose a specific browser binary.
  64     """
  65     try:
  66       self.SetUp(finder_options)
  67       self._PerformNavigations()
  68     finally:
  69       self.TearDown()
  70
  71   def GetUrlIterator(self):
  72     """Gets URLs for the browser to navigate to.
  73
  74     Intended for subclass override.
  75
  76     Returns:
  77       An iterator whose elements are urls to be navigated to.
  78     """
  79     raise NotImplementedError()
  80
  81   def ShouldExitAfterBatchNavigation(self):
  82     """Returns a boolean indicating whether profile extension is finished.
  83
  84     Intended for subclass override.
  85     """
  86     raise NotImplementedError()
  87
  88   def SetUp(self, finder_options):
  89     """Finds the browser, starts the browser, and opens the requisite number of
  90     tabs.
  91
  92     Can be overridden by subclasses. Subclasses must call the super class
  93     implementation.
  94     """
  95     self._profile_path = finder_options.output_profile_path
  96     possible_browser = self._GetPossibleBrowser(finder_options)
  97
  98     assert possible_browser.supports_tab_control
  99     assert (platform.GetHostPlatform().GetOSName() in
 100         ["win", "mac", "linux"])
 101     self._browser = possible_browser.Create(finder_options)
 102
 103   def TearDown(self):
 104     """Teardown that is guaranteed to be executed before the instance is
 105     destroyed.
 106
 107     Can be overridden by subclasses. Subclasses must call the super class
 108     implementation.
 109     """
 110     if self._browser:
 111       self._browser.Close()
 112       self._browser = None
 113
 114   def CleanUpAfterBatchNavigation(self):
 115     """A hook for subclasses to perform cleanup after each batch of
 116     navigations.
 117
 118     Can be overridden by subclasses.
 119     """
 120     pass
 121
 122   @property
 123   def profile_path(self):
 124     return self._profile_path
 125
 126   def _RefreshNavigationTabs(self):
 127     """Updates the member self._navigation_tabs to contain self._NUM_TABS
 128     elements, each of which is not crashed. The crashed tabs are intentionally
 129     leaked, since Telemetry doesn't have a good way of killing crashed tabs.
 130
 131     It is also possible for a tab to be stalled in an infinite JavaScript loop.
 132     These tabs will be in self._browser.tabs, but not in self._navigation_tabs.
 133     There is no way to kill these tabs, so they are also leaked. This method is
 134     careful to only use tabs in self._navigation_tabs, or newly created tabs.
 135     """
 136     live_tabs = [tab for tab in self._navigation_tabs if tab.IsAlive()]
 137     self._navigation_tabs = live_tabs
 138
 139     while len(self._navigation_tabs) < self._NUM_TABS:
 140       self._navigation_tabs.append(self._browser.tabs.New())
 141
 142   def _RemoveNavigationTab(self, tab):
 143     """Removes a tab which is no longer in a useable state from
 144     self._navigation_tabs. The tab is not removed from self._browser.tabs,
 145     since there is no guarantee that the tab can be safely removed."""
 146     self._navigation_tabs.remove(tab)
 147
 148   def _GetPossibleBrowser(self, finder_options):
 149     """Return a possible_browser with the given options."""
 150     possible_browser = browser_finder.FindBrowser(finder_options)
 151     if not possible_browser:
 152       raise browser_finder_exceptions.BrowserFinderException(
 153           'No browser found.\n\nAvailable browsers:\n%s\n' %
 154           '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
 155     finder_options.browser_options.browser_type = (
 156         possible_browser.browser_type)
 157
 158     return possible_browser
 159
 160   def _RetrieveTabUrl(self, tab, timeout):
 161     """Retrives the URL of the tab."""
 162     try:
 163       return tab.EvaluateJavaScript('document.URL', timeout)
 164     except exceptions.Error:
 165       return None
 166
 167   def _WaitForUrlToChange(self, tab, initial_url, timeout):
 168     """Waits for the tab to navigate away from its initial url."""
 169     end_time = time.time() + timeout
 170     while True:
 171       seconds_to_wait = end_time - time.time()
 172       seconds_to_wait = max(0, seconds_to_wait)
 173
 174       if seconds_to_wait == 0:
 175         break
 176
 177       current_url = self._RetrieveTabUrl(tab, seconds_to_wait)
 178       if current_url != initial_url:
 179         break
 180
 181       # Retrieving the current url is a non-trivial operation. Add a small
 182       # sleep here to prevent this method from contending with the actual
 183       # navigation.
 184       time.sleep(0.01)
 185
 186   def _BatchNavigateTabs(self, batch):
 187     """Performs a batch of tab navigations with minimal delay.
 188
 189     Args:
 190       batch: A list of tuples (tab, url).
 191
 192     Returns:
 193       A list of tuples (tab, initial_url). |initial_url| is the url of the
 194       |tab| prior to a navigation command being sent to it.
 195     """
 196     timeout_in_seconds = 0
 197
 198     queued_tabs = []
 199     for tab, url in batch:
 200       initial_url = self._RetrieveTabUrl(tab,
 201           self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS)
 202
 203       try:
 204         tab.Navigate(url, None, timeout_in_seconds)
 205       except exceptions.Error:
 206         # We expect a time out. It's possible for other problems to arise, but
 207         # this method is not responsible for dealing with them. Ignore all
 208         # exceptions.
 209         pass
 210
 211       queued_tabs.append((tab, initial_url))
 212     return queued_tabs
 213
 214   def _WaitForQueuedTabsToLoad(self, queued_tabs):
 215     """Waits for all the batch navigated tabs to finish loading.
 216
 217     Args:
 218       queued_tabs: A list of tuples (tab, initial_url). Each tab is guaranteed
 219       to have already been sent a navigation command.
 220     """
 221     end_time = time.time() + self._BATCH_PAGE_LOAD_TIMEOUT_IN_SECONDS
 222     for tab, initial_url in queued_tabs:
 223       seconds_to_wait = end_time - time.time()
 224       seconds_to_wait = max(0, seconds_to_wait)
 225
 226       if seconds_to_wait == 0:
 227         break
 228
 229       # Since we don't wait any time for the tab url navigation to commit, it's
 230       # possible that the tab hasn't started navigating yet.
 231       self._WaitForUrlToChange(tab, initial_url, seconds_to_wait)
 232
 233       seconds_to_wait = end_time - time.time()
 234       seconds_to_wait = max(0, seconds_to_wait)
 235
 236       try:
 237         tab.WaitForDocumentReadyStateToBeComplete(seconds_to_wait)
 238       except exceptions.TimeoutException:
 239         # Ignore time outs.
 240         pass
 241       except exceptions.Error:
 242         # If any error occurs, remove the tab. it's probably in an
 243         # unrecoverable state.
 244         self._RemoveNavigationTab(tab)
 245
 246   def _GetUrlsToNavigate(self, url_iterator):
 247     """Returns an array of urls to navigate to, given a url_iterator."""
 248     urls = []
 249     for _ in xrange(self._NUM_TABS):
 250       try:
 251         urls.append(url_iterator.next())
 252       except StopIteration:
 253         break
 254     return urls
 255
 256   def _PerformNavigations(self):
 257     """Repeatedly fetches a batch of urls, and navigates to those urls. This
 258     will run until an empty batch is returned, or
 259     ShouldExitAfterBatchNavigation() returns True.
 260     """
 261     url_iterator = self.GetUrlIterator()
 262     while True:
 263       self._RefreshNavigationTabs()
 264       urls = self._GetUrlsToNavigate(url_iterator)
 265
 266       if len(urls) == 0:
 267         break
 268
 269       batch = []
 270       for i in range(len(urls)):
 271         url = urls[i]
 272         tab = self._navigation_tabs[i]
 273         batch.append((tab, url))
 274
 275       queued_tabs = self._BatchNavigateTabs(batch)
 276       self._WaitForQueuedTabsToLoad(queued_tabs)
 277
 278       self.CleanUpAfterBatchNavigation()
 279
 280       if self.ShouldExitAfterBatchNavigation():
 281         break