tools/perf/profile_creators/fast_navigation_profile_extender.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import time
   5
   6 from profile_creators import profile_extender
   7 from telemetry.core import exceptions
   8
   9
  10 class FastNavigationProfileExtender(profile_extender.ProfileExtender):
  11   """Extends a Chrome profile.
  12
  13   This class creates or extends an existing profile by performing a set of tab
  14   navigations in large batches. This is accomplished by opening a large number
  15   of tabs, simultaneously navigating all the tabs, and then waiting for all the
  16   tabs to load. This provides two benefits:
  17     - Takes advantage of the high number of logical cores on modern CPUs.
  18     - The total time spent waiting for navigations to time out scales linearly
  19       with the number of batches, but does not scale with the size of the
  20       batch.
  21   """
  22   def __init__(self, finder_options, maximum_batch_size):
  23     """Initializer.
  24
  25     Args:
  26       maximum_batch_size: A positive integer indicating the number of tabs to
  27       simultaneously perform navigations.
  28     """
  29     super(FastNavigationProfileExtender, self).__init__(finder_options)
  30
  31     # The instance keeps a list of Tabs that can be navigated successfully.
  32     # This means that the Tab is not crashed, and is processing JavaScript in a
  33     # timely fashion.
  34     self._navigation_tabs = []
  35
  36     # The number of tabs to use.
  37     self._NUM_TABS = maximum_batch_size
  38
  39     # The amount of time to wait for a batch of pages to finish loading.
  40     self._BATCH_PAGE_LOAD_TIMEOUT_IN_SECONDS = 10
  41
  42     # The default amount of time to wait for the retrieval of the URL of a tab.
  43     self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS = 1
  44
  45   def Run(self):
  46     """Superclass override."""
  47     try:
  48       self.SetUpBrowser()
  49       self._PerformNavigations()
  50     finally:
  51       self.TearDownBrowser()
  52
  53   def GetUrlIterator(self):
  54     """Gets URLs for the browser to navigate to.
  55
  56     Intended for subclass override.
  57
  58     Returns:
  59       An iterator whose elements are urls to be navigated to.
  60     """
  61     raise NotImplementedError()
  62
  63   def ShouldExitAfterBatchNavigation(self):
  64     """Returns a boolean indicating whether profile extension is finished.
  65
  66     Intended for subclass override.
  67     """
  68     raise NotImplementedError()
  69
  70   def CleanUpAfterBatchNavigation(self):
  71     """A hook for subclasses to perform cleanup after each batch of
  72     navigations.
  73
  74     Can be overridden by subclasses.
  75     """
  76     pass
  77
  78   def _AddNewTab(self):
  79     """Adds a new tab to the browser."""
  80
  81     # Adding a new tab requires making a request over devtools. This can fail
  82     # for a variety of reasons. Retry 3 times.
  83     retry_count = 3
  84     for i in range(retry_count):
  85       try:
  86         self._navigation_tabs.append(self._browser.tabs.New())
  87       except exceptions.Error:
  88         if i == retry_count - 1:
  89           raise
  90       else:
  91         break
  92
  93   def _RefreshNavigationTabs(self):
  94     """Updates the member self._navigation_tabs to contain self._NUM_TABS
  95     elements, each of which is not crashed. The crashed tabs are intentionally
  96     leaked, since Telemetry doesn't have a good way of killing crashed tabs.
  97
  98     It is also possible for a tab to be stalled in an infinite JavaScript loop.
  99     These tabs will be in self.browser.tabs, but not in self._navigation_tabs.
 100     There is no way to kill these tabs, so they are also leaked. This method is
 101     careful to only use tabs in self._navigation_tabs, or newly created tabs.
 102     """
 103     live_tabs = [tab for tab in self._navigation_tabs if tab.IsAlive()]
 104     self._navigation_tabs = live_tabs
 105
 106     while len(self._navigation_tabs) < self._NUM_TABS:
 107       self._AddNewTab()
 108
 109   def _RemoveNavigationTab(self, tab):
 110     """Removes a tab which is no longer in a useable state from
 111     self._navigation_tabs. The tab is not removed from self.browser.tabs,
 112     since there is no guarantee that the tab can be safely removed."""
 113     self._navigation_tabs.remove(tab)
 114
 115   def _RetrieveTabUrl(self, tab, timeout):
 116     """Retrives the URL of the tab."""
 117     try:
 118       return tab.EvaluateJavaScript('document.URL', timeout)
 119     except exceptions.Error:
 120       return None
 121
 122   def _WaitForUrlToChange(self, tab, initial_url, timeout):
 123     """Waits for the tab to navigate away from its initial url."""
 124     end_time = time.time() + timeout
 125     while True:
 126       seconds_to_wait = end_time - time.time()
 127       seconds_to_wait = max(0, seconds_to_wait)
 128
 129       if seconds_to_wait == 0:
 130         break
 131
 132       current_url = self._RetrieveTabUrl(tab, seconds_to_wait)
 133       if current_url != initial_url:
 134         break
 135
 136       # Retrieving the current url is a non-trivial operation. Add a small
 137       # sleep here to prevent this method from contending with the actual
 138       # navigation.
 139       time.sleep(0.01)
 140
 141   def _BatchNavigateTabs(self, batch):
 142     """Performs a batch of tab navigations with minimal delay.
 143
 144     Args:
 145       batch: A list of tuples (tab, url).
 146
 147     Returns:
 148       A list of tuples (tab, initial_url). |initial_url| is the url of the
 149       |tab| prior to a navigation command being sent to it.
 150     """
 151     timeout_in_seconds = 0
 152
 153     queued_tabs = []
 154     for tab, url in batch:
 155       initial_url = self._RetrieveTabUrl(tab,
 156           self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS)
 157
 158       try:
 159         tab.Navigate(url, None, timeout_in_seconds)
 160       except exceptions.Error:
 161         # We expect a time out. It's possible for other problems to arise, but
 162         # this method is not responsible for dealing with them. Ignore all
 163         # exceptions.
 164         pass
 165
 166       queued_tabs.append((tab, initial_url))
 167     return queued_tabs
 168
 169   def _WaitForQueuedTabsToLoad(self, queued_tabs):
 170     """Waits for all the batch navigated tabs to finish loading.
 171
 172     Args:
 173       queued_tabs: A list of tuples (tab, initial_url). Each tab is guaranteed
 174       to have already been sent a navigation command.
 175     """
 176     end_time = time.time() + self._BATCH_PAGE_LOAD_TIMEOUT_IN_SECONDS
 177     for tab, initial_url in queued_tabs:
 178       seconds_to_wait = end_time - time.time()
 179       seconds_to_wait = max(0, seconds_to_wait)
 180
 181       if seconds_to_wait == 0:
 182         break
 183
 184       # Since we don't wait any time for the tab url navigation to commit, it's
 185       # possible that the tab hasn't started navigating yet.
 186       self._WaitForUrlToChange(tab, initial_url, seconds_to_wait)
 187
 188       seconds_to_wait = end_time - time.time()
 189       seconds_to_wait = max(0, seconds_to_wait)
 190
 191       try:
 192         tab.WaitForDocumentReadyStateToBeComplete(seconds_to_wait)
 193       except exceptions.TimeoutException:
 194         # Ignore time outs.
 195         pass
 196       except exceptions.Error:
 197         # If any error occurs, remove the tab. it's probably in an
 198         # unrecoverable state.
 199         self._RemoveNavigationTab(tab)
 200
 201   def _GetUrlsToNavigate(self, url_iterator):
 202     """Returns an array of urls to navigate to, given a url_iterator."""
 203     urls = []
 204     for _ in xrange(self._NUM_TABS):
 205       try:
 206         urls.append(url_iterator.next())
 207       except StopIteration:
 208         break
 209     return urls
 210
 211   def _PerformNavigations(self):
 212     """Repeatedly fetches a batch of urls, and navigates to those urls. This
 213     will run until an empty batch is returned, or
 214     ShouldExitAfterBatchNavigation() returns True.
 215     """
 216     url_iterator = self.GetUrlIterator()
 217     while True:
 218       self._RefreshNavigationTabs()
 219       urls = self._GetUrlsToNavigate(url_iterator)
 220
 221       if len(urls) == 0:
 222         break
 223
 224       batch = []
 225       for i in range(len(urls)):
 226         url = urls[i]
 227         tab = self._navigation_tabs[i]
 228         batch.append((tab, url))
 229
 230       queued_tabs = self._BatchNavigateTabs(batch)
 231       self._WaitForQueuedTabsToLoad(queued_tabs)
 232
 233       self.CleanUpAfterBatchNavigation()
 234
 235       if self.ShouldExitAfterBatchNavigation():
 236         break