tools/perf/metrics/speedindex.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import collections
   6 import logging
   7
   8 from telemetry.image_processing import image_util
   9 from telemetry.image_processing import rgba_color
  10 from telemetry.value import scalar
  11
  12 from metrics import Metric
  13
  14
  15 class SpeedIndexMetric(Metric):
  16   """The speed index metric is one way of measuring page load speed.
  17
  18   It is meant to approximate user perception of page load speed, and it
  19   is based on the amount of time that it takes to paint to the visual
  20   portion of the screen. It includes paint events that occur after the
  21   onload event, and it doesn't include time loading things off-screen.
  22
  23   This speed index metric is based on WebPageTest.org (WPT).
  24   For more info see: http://goo.gl/e7AH5l
  25   """
  26   def __init__(self):
  27     super(SpeedIndexMetric, self).__init__()
  28     self._impl = None
  29
  30   @classmethod
  31   def CustomizeBrowserOptions(cls, options):
  32     options.AppendExtraBrowserArgs('--disable-infobars')
  33
  34   def Start(self, _, tab):
  35     """Start recording events.
  36
  37     This method should be called in the WillNavigateToPage method of
  38     a PageTest, so that all the events can be captured. If it's called
  39     in DidNavigateToPage, that will be too late.
  40     """
  41     if not tab.video_capture_supported:
  42       return
  43     self._impl = VideoSpeedIndexImpl()
  44     self._impl.Start(tab)
  45
  46   def Stop(self, _, tab):
  47     """Stop recording."""
  48     if not tab.video_capture_supported:
  49       return
  50     assert self._impl, 'Must call Start() before Stop()'
  51     assert self.IsFinished(tab), 'Must wait for IsFinished() before Stop()'
  52     self._impl.Stop(tab)
  53
  54   # Optional argument chart_name is not in base class Metric.
  55   # pylint: disable=W0221
  56   def AddResults(self, tab, results, chart_name=None):
  57     """Calculate the speed index and add it to the results."""
  58     try:
  59       if tab.video_capture_supported:
  60         index = self._impl.CalculateSpeedIndex(tab)
  61         none_value_reason = None
  62       else:
  63         index = None
  64         none_value_reason = 'Video capture is not supported.'
  65     finally:
  66       self._impl = None  # Release the tab so that it can be disconnected.
  67
  68     results.AddValue(scalar.ScalarValue(
  69         results.current_page, '%s_speed_index' % chart_name, 'ms', index,
  70         description='Speed Index. This focuses on time when visible parts of '
  71                     'page are displayed and shows the time when the '
  72                     'first look is "almost" composed. If the contents of the '
  73                     'testing page are composed by only static resources, load '
  74                     'time can measure more accurately and speed index will be '
  75                     'smaller than the load time. On the other hand, If the '
  76                     'contents are composed by many XHR requests with small '
  77                     'main resource and javascript, speed index will be able to '
  78                     'get the features of performance more accurately than load '
  79                     'time because the load time will measure the time when '
  80                     'static resources are loaded. If you want to get more '
  81                     'detail, please refer to http://goo.gl/Rw3d5d. Currently '
  82                     'there are two implementations: for Android and for '
  83                     'Desktop. The Android version uses video capture; the '
  84                     'Desktop one uses paint events and has extra overhead to '
  85                     'catch paint events.', none_value_reason=none_value_reason))
  86
  87   def IsFinished(self, tab):
  88     """Decide whether the recording should be stopped.
  89
  90     A page may repeatedly request resources in an infinite loop; a timeout
  91     should be placed in any measurement that uses this metric, e.g.:
  92       def IsDone():
  93         return self._speedindex.IsFinished(tab)
  94       util.WaitFor(IsDone, 60)
  95
  96     Returns:
  97       True if 2 seconds have passed since last resource received, false
  98       otherwise.
  99     """
 100     return tab.HasReachedQuiescence()
 101
 102
 103 class SpeedIndexImpl(object):
 104
 105   def Start(self, tab):
 106     raise NotImplementedError()
 107
 108   def Stop(self, tab):
 109     raise NotImplementedError()
 110
 111   def GetTimeCompletenessList(self, tab):
 112     """Returns a list of time to visual completeness tuples.
 113
 114     In the WPT PHP implementation, this is also called 'visual progress'.
 115     """
 116     raise NotImplementedError()
 117
 118   def CalculateSpeedIndex(self, tab):
 119     """Calculate the speed index.
 120
 121     The speed index number conceptually represents the number of milliseconds
 122     that the page was "visually incomplete". If the page were 0% complete for
 123     1000 ms, then the score would be 1000; if it were 0% complete for 100 ms
 124     then 90% complete (ie 10% incomplete) for 900 ms, then the score would be
 125     1.0*100 + 0.1*900 = 190.
 126
 127     Returns:
 128       A single number, milliseconds of visual incompleteness.
 129     """
 130     time_completeness_list = self.GetTimeCompletenessList(tab)
 131     prev_completeness = 0.0
 132     speed_index = 0.0
 133     prev_time = time_completeness_list[0][0]
 134     for time, completeness in time_completeness_list:
 135       # Add the incemental value for the interval just before this event.
 136       elapsed_time = time - prev_time
 137       incompleteness = (1.0 - prev_completeness)
 138       speed_index += elapsed_time * incompleteness
 139
 140       # Update variables for next iteration.
 141       prev_completeness = completeness
 142       prev_time = time
 143     return int(speed_index)
 144
 145
 146 class VideoSpeedIndexImpl(SpeedIndexImpl):
 147
 148   def __init__(self, image_util_module=image_util):
 149     # Allow image_util to be passed in so we can fake it out for testing.
 150     super(VideoSpeedIndexImpl, self).__init__()
 151     self._time_completeness_list = None
 152     self._image_util_module = image_util_module
 153
 154   def Start(self, tab):
 155     assert tab.video_capture_supported
 156     # Blank out the current page so it doesn't count towards the new page's
 157     # completeness.
 158     tab.Highlight(rgba_color.WHITE)
 159     # TODO(tonyg): Bitrate is arbitrary here. Experiment with screen capture
 160     # overhead vs. speed index accuracy and set the bitrate appropriately.
 161     tab.StartVideoCapture(min_bitrate_mbps=4)
 162
 163   def Stop(self, tab):
 164     # Ignore white because Chrome may blank out the page during load and we want
 165     # that to count as 0% complete. Relying on this fact, we also blank out the
 166     # previous page to white. The tolerance of 8 experimentally does well with
 167     # video capture at 4mbps. We should keep this as low as possible with
 168     # supported video compression settings.
 169     video_capture = tab.StopVideoCapture()
 170     histograms = [(time, self._image_util_module.GetColorHistogram(
 171                        image, ignore_color=rgba_color.WHITE, tolerance=8))
 172                   for time, image in video_capture.GetVideoFrameIter()]
 173
 174     start_histogram = histograms[0][1]
 175     final_histogram = histograms[-1][1]
 176     total_distance = start_histogram.Distance(final_histogram)
 177
 178     def FrameProgress(histogram):
 179       if total_distance == 0:
 180         if histogram.Distance(final_histogram) == 0:
 181           return 1.0
 182         else:
 183           return 0.0
 184       return 1 - histogram.Distance(final_histogram) / total_distance
 185
 186     self._time_completeness_list = [(time, FrameProgress(hist))
 187                                     for time, hist in histograms]
 188
 189   def GetTimeCompletenessList(self, tab):
 190     assert self._time_completeness_list, 'Must call Stop() first.'
 191     return self._time_completeness_list