tools/perf/profile_creators/fast_navigation_profile_extender.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import time
   5
   6 from profile_creators import profile_extender
   7 from telemetry.core import exceptions
   8 from telemetry.core import util
   9
  10
  11 class FastNavigationProfileExtender(profile_extender.ProfileExtender):
  12   """Extends a Chrome profile.
  13
  14   This class creates or extends an existing profile by performing a set of tab
  15   navigations in large batches. This is accomplished by opening a large number
  16   of tabs, simultaneously navigating all the tabs, and then waiting for all the
  17   tabs to load. This provides two benefits:
  18     - Takes advantage of the high number of logical cores on modern CPUs.
  19     - The total time spent waiting for navigations to time out scales linearly
  20       with the number of batches, but does not scale with the size of the
  21       batch.
  22   """
  23   def __init__(self, finder_options, maximum_batch_size):
  24     """Initializer.
  25
  26     Args:
  27       maximum_batch_size: A positive integer indicating the number of tabs to
  28       simultaneously perform navigations.
  29     """
  30     super(FastNavigationProfileExtender, self).__init__(finder_options)
  31
  32     # The instance keeps a list of Tabs that can be navigated successfully.
  33     # This means that the Tab is not crashed, and is processing JavaScript in a
  34     # timely fashion.
  35     self._navigation_tabs = []
  36
  37     # The number of tabs to use.
  38     self._NUM_TABS = maximum_batch_size
  39
  40     # The amount of additional time to wait for a batch of pages to finish
  41     # loading for each page in the batch.
  42     self._BATCH_TIMEOUT_PER_PAGE_IN_SECONDS = 20
  43
  44     # The amount of time to wait for a page to quiesce. Some pages will never
  45     # quiesce.
  46     self._TIME_TO_WAIT_FOR_PAGE_TO_QUIESCE_IN_SECONDS = 10
  47
  48   def Run(self):
  49     """Superclass override."""
  50     try:
  51       self.SetUpBrowser()
  52       self._PerformNavigations()
  53     finally:
  54       self.TearDownBrowser()
  55
  56     # When there hasn't been an exception, verify that the profile was
  57     # correctly extended.
  58     # TODO(erikchen): I've intentionally omitted my implementation of
  59     # VerifyProfileWasExtended() in small_profile_extender, since the profile
  60     # is not being correctly extended. http://crbug.com/484833
  61     # http://crbug.com/484880
  62     self.VerifyProfileWasExtended()
  63
  64   def VerifyProfileWasExtended(self):
  65     """Verifies that the profile was correctly extended.
  66
  67     Can be overridden by subclasses.
  68     """
  69     pass
  70
  71   def GetUrlIterator(self):
  72     """Gets URLs for the browser to navigate to.
  73
  74     Intended for subclass override.
  75
  76     Returns:
  77       An iterator whose elements are urls to be navigated to.
  78     """
  79     raise NotImplementedError()
  80
  81   def ShouldExitAfterBatchNavigation(self):
  82     """Returns a boolean indicating whether profile extension is finished.
  83
  84     Intended for subclass override.
  85     """
  86     raise NotImplementedError()
  87
  88   def CleanUpAfterBatchNavigation(self):
  89     """A hook for subclasses to perform cleanup after each batch of
  90     navigations.
  91
  92     Can be overridden by subclasses.
  93     """
  94     pass
  95
  96   def _RefreshNavigationTabs(self):
  97     """Updates the member self._navigation_tabs to contain self._NUM_TABS
  98     elements, each of which is not crashed. The crashed tabs are intentionally
  99     leaked, since Telemetry doesn't have a good way of killing crashed tabs.
 100
 101     It is also possible for a tab to be stalled in an infinite JavaScript loop.
 102     These tabs will be in self.browser.tabs, but not in self._navigation_tabs.
 103     There is no way to kill these tabs, so they are also leaked. This method is
 104     careful to only use tabs in self._navigation_tabs, or newly created tabs.
 105     """
 106     live_tabs = [tab for tab in self._navigation_tabs if tab.IsAlive()]
 107     self._navigation_tabs = live_tabs
 108
 109     while len(self._navigation_tabs) < self._NUM_TABS:
 110       self._navigation_tabs.append(self._browser.tabs.New())
 111
 112   def _RemoveNavigationTab(self, tab):
 113     """Removes a tab which is no longer in a useable state from
 114     self._navigation_tabs. The tab is not removed from self.browser.tabs,
 115     since there is no guarantee that the tab can be safely removed."""
 116     self._navigation_tabs.remove(tab)
 117
 118   def _RetrieveTabUrl(self, tab, timeout):
 119     """Retrives the URL of the tab."""
 120     # TODO(erikchen): Use tab.url instead, which talks to the browser process
 121     # instead of the renderer process. http://crbug.com/486119
 122     return tab.EvaluateJavaScript('document.URL', timeout)
 123
 124   def _WaitForUrlToChange(self, tab, initial_url, end_time):
 125     """Waits for the tab to navigate away from its initial url.
 126
 127     If time.time() is larger than end_time, the function does nothing.
 128     Otherwise, the function tries to return no later than end_time.
 129     """
 130     while True:
 131       seconds_to_wait = end_time - time.time()
 132       if seconds_to_wait <= 0:
 133         break
 134
 135       current_url = self._RetrieveTabUrl(tab, seconds_to_wait)
 136       if current_url != initial_url and current_url != "":
 137         break
 138
 139       # Retrieving the current url is a non-trivial operation. Add a small
 140       # sleep here to prevent this method from contending with the actual
 141       # navigation.
 142       time.sleep(0.01)
 143
 144   def _WaitForTabToBeReady(self, tab, end_time):
 145     """Waits for the tab to be ready.
 146
 147     If time.time() is larger than end_time, the function does nothing.
 148     Otherwise, the function tries to return no later than end_time.
 149     """
 150     seconds_to_wait = end_time - time.time()
 151     if seconds_to_wait <= 0:
 152       return
 153     tab.WaitForDocumentReadyStateToBeComplete(seconds_to_wait)
 154
 155     # Wait up to 10 seconds for the page to quiesce. If the page hasn't
 156     # quiesced in 10 seconds, it will probably never quiesce.
 157     seconds_to_wait = end_time - time.time()
 158     seconds_to_wait = max(0, seconds_to_wait)
 159     try:
 160       util.WaitFor(tab.HasReachedQuiescence, seconds_to_wait)
 161     except exceptions.TimeoutException:
 162       pass
 163
 164   def _BatchNavigateTabs(self, batch):
 165     """Performs a batch of tab navigations with minimal delay.
 166
 167     Args:
 168       batch: A list of tuples (tab, url).
 169
 170     Returns:
 171       A list of tuples (tab, initial_url). |initial_url| is the url of the
 172       |tab| prior to a navigation command being sent to it.
 173     """
 174     # Attempting to pass in a timeout of 0 seconds results in a synchronous
 175     # socket error from the websocket library. Pass in a very small timeout
 176     # instead so that the websocket library raises a Timeout exception. This
 177     # prevents the logic from accidentally catching different socket
 178     # exceptions.
 179     timeout_in_seconds = 0.01
 180
 181     queued_tabs = []
 182     for tab, url in batch:
 183       initial_url = self._RetrieveTabUrl(tab, 20)
 184       try:
 185         tab.Navigate(url, None, timeout_in_seconds)
 186       except exceptions.TimeoutException:
 187         # We expect to receive a timeout exception, since we're not waiting for
 188         # the navigation to complete.
 189         pass
 190       queued_tabs.append((tab, initial_url))
 191     return queued_tabs
 192
 193   def _WaitForQueuedTabsToLoad(self, queued_tabs):
 194     """Waits for all the batch navigated tabs to finish loading.
 195
 196     Args:
 197       queued_tabs: A list of tuples (tab, initial_url). Each tab is guaranteed
 198       to have already been sent a navigation command.
 199     """
 200     total_batch_timeout = (len(queued_tabs) *
 201         self._BATCH_TIMEOUT_PER_PAGE_IN_SECONDS)
 202     end_time = time.time() + total_batch_timeout
 203     for tab, initial_url in queued_tabs:
 204       # Since we didn't wait any time for the tab url navigation to commit, it's
 205       # possible that the tab hasn't started navigating yet.
 206       self._WaitForUrlToChange(tab, initial_url, end_time)
 207       self._WaitForTabToBeReady(tab, end_time)
 208
 209   def _GetUrlsToNavigate(self, url_iterator):
 210     """Returns an array of urls to navigate to, given a url_iterator."""
 211     urls = []
 212     for _ in xrange(self._NUM_TABS):
 213       try:
 214         urls.append(url_iterator.next())
 215       except StopIteration:
 216         break
 217     return urls
 218
 219   def _PerformNavigations(self):
 220     """Repeatedly fetches a batch of urls, and navigates to those urls. This
 221     will run until an empty batch is returned, or
 222     ShouldExitAfterBatchNavigation() returns True.
 223     """
 224     url_iterator = self.GetUrlIterator()
 225     while True:
 226       self._RefreshNavigationTabs()
 227       urls = self._GetUrlsToNavigate(url_iterator)
 228
 229       if len(urls) == 0:
 230         break
 231
 232       batch = []
 233       for i in range(len(urls)):
 234         url = urls[i]
 235         tab = self._navigation_tabs[i]
 236         batch.append((tab, url))
 237
 238       queued_tabs = self._BatchNavigateTabs(batch)
 239       self._WaitForQueuedTabsToLoad(queued_tabs)
 240
 241       self.CleanUpAfterBatchNavigation()
 242
 243       if self.ShouldExitAfterBatchNavigation():
 244         break