tools/perf/metrics/speedindex.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import collections
   6 import logging
   7
   8 from metrics import Metric
   9 from telemetry.image_processing import image_util
  10 from telemetry.image_processing import rgba_color
  11 from telemetry.value import scalar
  12
  13
  14 class SpeedIndexMetric(Metric):
  15   """The speed index metric is one way of measuring page load speed.
  16
  17   It is meant to approximate user perception of page load speed, and it
  18   is based on the amount of time that it takes to paint to the visual
  19   portion of the screen. It includes paint events that occur after the
  20   onload event, and it doesn't include time loading things off-screen.
  21
  22   This speed index metric is based on WebPageTest.org (WPT).
  23   For more info see: http://goo.gl/e7AH5l
  24   """
  25   def __init__(self):
  26     super(SpeedIndexMetric, self).__init__()
  27     self._impl = None
  28
  29   @classmethod
  30   def CustomizeBrowserOptions(cls, options):
  31     options.AppendExtraBrowserArgs('--disable-infobars')
  32
  33   def Start(self, _, tab):
  34     """Start recording events.
  35
  36     This method should be called in the WillNavigateToPage method of
  37     a PageTest, so that all the events can be captured. If it's called
  38     in DidNavigateToPage, that will be too late.
  39     """
  40     if not tab.video_capture_supported:
  41       return
  42     self._impl = VideoSpeedIndexImpl()
  43     self._impl.Start(tab)
  44
  45   def Stop(self, _, tab):
  46     """Stop recording."""
  47     if not tab.video_capture_supported:
  48       return
  49     assert self._impl, 'Must call Start() before Stop()'
  50     assert self.IsFinished(tab), 'Must wait for IsFinished() before Stop()'
  51     self._impl.Stop(tab)
  52
  53   # Optional argument chart_name is not in base class Metric.
  54   # pylint: disable=W0221
  55   def AddResults(self, tab, results, chart_name=None):
  56     """Calculate the speed index and add it to the results."""
  57     try:
  58       if tab.video_capture_supported:
  59         index = self._impl.CalculateSpeedIndex(tab)
  60         none_value_reason = None
  61       else:
  62         index = None
  63         none_value_reason = 'Video capture is not supported.'
  64     finally:
  65       self._impl = None  # Release the tab so that it can be disconnected.
  66
  67     results.AddValue(scalar.ScalarValue(
  68         results.current_page, '%s_speed_index' % chart_name, 'ms', index,
  69         description='Speed Index. This focuses on time when visible parts of '
  70                     'page are displayed and shows the time when the '
  71                     'first look is "almost" composed. If the contents of the '
  72                     'testing page are composed by only static resources, load '
  73                     'time can measure more accurately and speed index will be '
  74                     'smaller than the load time. On the other hand, If the '
  75                     'contents are composed by many XHR requests with small '
  76                     'main resource and javascript, speed index will be able to '
  77                     'get the features of performance more accurately than load '
  78                     'time because the load time will measure the time when '
  79                     'static resources are loaded. If you want to get more '
  80                     'detail, please refer to http://goo.gl/Rw3d5d. Currently '
  81                     'there are two implementations: for Android and for '
  82                     'Desktop. The Android version uses video capture; the '
  83                     'Desktop one uses paint events and has extra overhead to '
  84                     'catch paint events.', none_value_reason=none_value_reason))
  85
  86   def IsFinished(self, tab):
  87     """Decide whether the recording should be stopped.
  88
  89     A page may repeatedly request resources in an infinite loop; a timeout
  90     should be placed in any measurement that uses this metric, e.g.:
  91       def IsDone():
  92         return self._speedindex.IsFinished(tab)
  93       util.WaitFor(IsDone, 60)
  94
  95     Returns:
  96       True if 2 seconds have passed since last resource received, false
  97       otherwise.
  98     """
  99     return tab.HasReachedQuiescence()
 100
 101
 102 class SpeedIndexImpl(object):
 103
 104   def Start(self, tab):
 105     raise NotImplementedError()
 106
 107   def Stop(self, tab):
 108     raise NotImplementedError()
 109
 110   def GetTimeCompletenessList(self, tab):
 111     """Returns a list of time to visual completeness tuples.
 112
 113     In the WPT PHP implementation, this is also called 'visual progress'.
 114     """
 115     raise NotImplementedError()
 116
 117   def CalculateSpeedIndex(self, tab):
 118     """Calculate the speed index.
 119
 120     The speed index number conceptually represents the number of milliseconds
 121     that the page was "visually incomplete". If the page were 0% complete for
 122     1000 ms, then the score would be 1000; if it were 0% complete for 100 ms
 123     then 90% complete (ie 10% incomplete) for 900 ms, then the score would be
 124     1.0*100 + 0.1*900 = 190.
 125
 126     Returns:
 127       A single number, milliseconds of visual incompleteness.
 128     """
 129     time_completeness_list = self.GetTimeCompletenessList(tab)
 130     prev_completeness = 0.0
 131     speed_index = 0.0
 132     prev_time = time_completeness_list[0][0]
 133     for time, completeness in time_completeness_list:
 134       # Add the incemental value for the interval just before this event.
 135       elapsed_time = time - prev_time
 136       incompleteness = (1.0 - prev_completeness)
 137       speed_index += elapsed_time * incompleteness
 138
 139       # Update variables for next iteration.
 140       prev_completeness = completeness
 141       prev_time = time
 142     return int(speed_index)
 143
 144
 145 class VideoSpeedIndexImpl(SpeedIndexImpl):
 146
 147   def __init__(self, image_util_module=image_util):
 148     # Allow image_util to be passed in so we can fake it out for testing.
 149     super(VideoSpeedIndexImpl, self).__init__()
 150     self._time_completeness_list = None
 151     self._image_util_module = image_util_module
 152
 153   def Start(self, tab):
 154     assert tab.video_capture_supported
 155     # Blank out the current page so it doesn't count towards the new page's
 156     # completeness.
 157     tab.Highlight(rgba_color.WHITE)
 158     # TODO(tonyg): Bitrate is arbitrary here. Experiment with screen capture
 159     # overhead vs. speed index accuracy and set the bitrate appropriately.
 160     tab.StartVideoCapture(min_bitrate_mbps=4)
 161
 162   def Stop(self, tab):
 163     # Ignore white because Chrome may blank out the page during load and we want
 164     # that to count as 0% complete. Relying on this fact, we also blank out the
 165     # previous page to white. The tolerance of 8 experimentally does well with
 166     # video capture at 4mbps. We should keep this as low as possible with
 167     # supported video compression settings.
 168     video_capture = tab.StopVideoCapture()
 169     histograms = [(time, self._image_util_module.GetColorHistogram(
 170                        image, ignore_color=rgba_color.WHITE, tolerance=8))
 171                   for time, image in video_capture.GetVideoFrameIter()]
 172
 173     start_histogram = histograms[0][1]
 174     final_histogram = histograms[-1][1]
 175     total_distance = start_histogram.Distance(final_histogram)
 176
 177     def FrameProgress(histogram):
 178       if total_distance == 0:
 179         if histogram.Distance(final_histogram) == 0:
 180           return 1.0
 181         else:
 182           return 0.0
 183       return 1 - histogram.Distance(final_histogram) / total_distance
 184
 185     self._time_completeness_list = [(time, FrameProgress(hist))
 186                                     for time, hist in histograms]
 187
 188   def GetTimeCompletenessList(self, tab):
 189     assert self._time_completeness_list, 'Must call Stop() first.'
 190     return self._time_completeness_list