tools/perf/metrics/speedindex.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import collections
   6
   7 from metrics import Metric
   8 from telemetry.core import bitmap
   9 from telemetry.value import scalar
  10
  11
  12 class SpeedIndexMetric(Metric):
  13   """The speed index metric is one way of measuring page load speed.
  14
  15   It is meant to approximate user perception of page load speed, and it
  16   is based on the amount of time that it takes to paint to the visual
  17   portion of the screen. It includes paint events that occur after the
  18   onload event, and it doesn't include time loading things off-screen.
  19
  20   This speed index metric is based on WebPageTest.org (WPT).
  21   For more info see: http://goo.gl/e7AH5l
  22   """
  23   def __init__(self):
  24     super(SpeedIndexMetric, self).__init__()
  25     self._impl = None
  26
  27   @classmethod
  28   def CustomizeBrowserOptions(cls, options):
  29     options.AppendExtraBrowserArgs('--disable-infobars')
  30
  31   def Start(self, _, tab):
  32     """Start recording events.
  33
  34     This method should be called in the WillNavigateToPage method of
  35     a PageTest, so that all the events can be captured. If it's called
  36     in DidNavigateToPage, that will be too late.
  37     """
  38     self._impl = (VideoSpeedIndexImpl() if tab.video_capture_supported else
  39                   PaintRectSpeedIndexImpl())
  40     self._impl.Start(tab)
  41
  42   def Stop(self, _, tab):
  43     """Stop timeline recording."""
  44     assert self._impl, 'Must call Start() before Stop()'
  45     assert self.IsFinished(tab), 'Must wait for IsFinished() before Stop()'
  46     self._impl.Stop(tab)
  47
  48   # Optional argument chart_name is not in base class Metric.
  49   # pylint: disable=W0221
  50   def AddResults(self, tab, results, chart_name=None):
  51     """Calculate the speed index and add it to the results."""
  52     index = self._impl.CalculateSpeedIndex(tab)
  53     # Release the tab so that it can be disconnected.
  54     self._impl = None
  55     results.AddValue(scalar.ScalarValue(
  56         results.current_page, '%s_speed_index' % chart_name, 'ms', index,
  57         description='Speed Index. This focuses on time when visible parts of '
  58                     'page are displayed and shows the time when the '
  59                     'first look is "almost" composed. If the contents of the '
  60                     'testing page are composed by only static resources, load '
  61                     'time can measure more accurately and speed index will be '
  62                     'smaller than the load time. On the other hand, If the '
  63                     'contents are composed by many XHR requests with small '
  64                     'main resource and javascript, speed index will be able to '
  65                     'get the features of performance more accurately than load '
  66                     'time because the load time will measure the time when '
  67                     'static resources are loaded. If you want to get more '
  68                     'detail, please refer to http://goo.gl/Rw3d5d. Currently '
  69                     'there are two implementations: for Android and for '
  70                     'Desktop. The Android version uses video capture; the '
  71                     'Desktop one uses paint events and has extra overhead to '
  72                     'catch paint events.'))
  73
  74   def IsFinished(self, tab):
  75     """Decide whether the timeline recording should be stopped.
  76
  77     When the timeline recording is stopped determines which paint events
  78     are used in the speed index metric calculation. In general, the recording
  79     should continue if there has just been some data received, because
  80     this suggests that painting may continue.
  81
  82     A page may repeatedly request resources in an infinite loop; a timeout
  83     should be placed in any measurement that uses this metric, e.g.:
  84       def IsDone():
  85         return self._speedindex.IsFinished(tab)
  86       util.WaitFor(IsDone, 60)
  87
  88     Returns:
  89       True if 2 seconds have passed since last resource received, false
  90       otherwise.
  91     """
  92     return tab.HasReachedQuiescence()
  93
  94
  95 class SpeedIndexImpl(object):
  96
  97   def Start(self, tab):
  98     raise NotImplementedError()
  99
 100   def Stop(self, tab):
 101     raise NotImplementedError()
 102
 103   def GetTimeCompletenessList(self, tab):
 104     """Returns a list of time to visual completeness tuples.
 105
 106     In the WPT PHP implementation, this is also called 'visual progress'.
 107     """
 108     raise NotImplementedError()
 109
 110   def CalculateSpeedIndex(self, tab):
 111     """Calculate the speed index.
 112
 113     The speed index number conceptually represents the number of milliseconds
 114     that the page was "visually incomplete". If the page were 0% complete for
 115     1000 ms, then the score would be 1000; if it were 0% complete for 100 ms
 116     then 90% complete (ie 10% incomplete) for 900 ms, then the score would be
 117     1.0*100 + 0.1*900 = 190.
 118
 119     Returns:
 120       A single number, milliseconds of visual incompleteness.
 121     """
 122     time_completeness_list = self.GetTimeCompletenessList(tab)
 123     prev_completeness = 0.0
 124     speed_index = 0.0
 125     prev_time = time_completeness_list[0][0]
 126     for time, completeness in time_completeness_list:
 127       # Add the incemental value for the interval just before this event.
 128       elapsed_time = time - prev_time
 129       incompleteness = (1.0 - prev_completeness)
 130       speed_index += elapsed_time * incompleteness
 131
 132       # Update variables for next iteration.
 133       prev_completeness = completeness
 134       prev_time = time
 135     return int(speed_index)
 136
 137
 138 class VideoSpeedIndexImpl(SpeedIndexImpl):
 139
 140   def __init__(self):
 141     super(VideoSpeedIndexImpl, self).__init__()
 142     self._time_completeness_list = None
 143
 144   def Start(self, tab):
 145     assert tab.video_capture_supported
 146     # Blank out the current page so it doesn't count towards the new page's
 147     # completeness.
 148     tab.Highlight(bitmap.WHITE)
 149     # TODO(tonyg): Bitrate is arbitrary here. Experiment with screen capture
 150     # overhead vs. speed index accuracy and set the bitrate appropriately.
 151     tab.StartVideoCapture(min_bitrate_mbps=4)
 152
 153   def Stop(self, tab):
 154     # Ignore white because Chrome may blank out the page during load and we want
 155     # that to count as 0% complete. Relying on this fact, we also blank out the
 156     # previous page to white. The tolerance of 8 experimentally does well with
 157     # video capture at 4mbps. We should keep this as low as possible with
 158     # supported video compression settings.
 159     video_capture = tab.StopVideoCapture()
 160     histograms = [(time, bmp.ColorHistogram(ignore_color=bitmap.WHITE,
 161                                             tolerance=8))
 162                   for time, bmp in video_capture.GetVideoFrameIter()]
 163
 164     start_histogram = histograms[0][1]
 165     final_histogram = histograms[-1][1]
 166     total_distance = start_histogram.Distance(final_histogram)
 167
 168     def FrameProgress(histogram):
 169       if total_distance == 0:
 170         if histogram.Distance(final_histogram) == 0:
 171           return 1.0
 172         else:
 173           return 0.0
 174       return 1 - histogram.Distance(final_histogram) / total_distance
 175
 176     self._time_completeness_list = [(time, FrameProgress(hist))
 177                                     for time, hist in histograms]
 178
 179   def GetTimeCompletenessList(self, tab):
 180     assert self._time_completeness_list, 'Must call Stop() first.'
 181     return self._time_completeness_list
 182
 183
 184 class PaintRectSpeedIndexImpl(SpeedIndexImpl):
 185
 186   def __init__(self):
 187     super(PaintRectSpeedIndexImpl, self).__init__()
 188
 189   def Start(self, tab):
 190     tab.StartTimelineRecording()
 191
 192   def Stop(self, tab):
 193     tab.StopTimelineRecording()
 194
 195   def GetTimeCompletenessList(self, tab):
 196     events = tab.timeline_model.GetAllEvents()
 197     viewport = self._GetViewportSize(tab)
 198     paint_events = self._IncludedPaintEvents(events)
 199     time_area_dict = self._TimeAreaDict(paint_events, viewport)
 200     total_area = sum(time_area_dict.values())
 201     assert total_area > 0.0, 'Total paint event area must be greater than 0.'
 202     completeness = 0.0
 203     time_completeness_list = []
 204
 205     # TODO(tonyg): This sets the start time to the start of the first paint
 206     # event. That can't be correct. The start time should be navigationStart.
 207     # Since the previous screen is not cleared at navigationStart, we should
 208     # probably assume the completeness is 0 until the first paint and add the
 209     # time of navigationStart as the start. We need to confirm what WPT does.
 210     time_completeness_list.append(
 211         (tab.timeline_model.GetAllEvents()[0].start, completeness))
 212
 213     for time, area in sorted(time_area_dict.items()):
 214       completeness += float(area) / total_area
 215       # Visual progress is rounded to the nearest percentage point as in WPT.
 216       time_completeness_list.append((time, round(completeness, 2)))
 217     return time_completeness_list
 218
 219   def _GetViewportSize(self, tab):
 220     """Returns dimensions of the viewport."""
 221     return tab.EvaluateJavaScript('[ window.innerWidth, window.innerHeight ]')
 222
 223   def _IncludedPaintEvents(self, events):
 224     """Get all events that are counted in the calculation of the speed index.
 225
 226     There's one category of paint event that's filtered out: paint events
 227     that occur before the first 'ResourceReceiveResponse' and 'Layout' events.
 228
 229     Previously in the WPT speed index, paint events that contain children paint
 230     events were also filtered out.
 231     """
 232     def FirstLayoutTime(events):
 233       """Get the start time of the first layout after a resource received."""
 234       has_received_response = False
 235       for event in events:
 236         if event.name == 'ResourceReceiveResponse':
 237           has_received_response = True
 238         elif has_received_response and event.name == 'Layout':
 239           return event.start
 240       assert False, 'There were no layout events after resource receive events.'
 241
 242     first_layout_time = FirstLayoutTime(events)
 243     paint_events = [e for e in events
 244                     if e.start >= first_layout_time and e.name == 'Paint']
 245     return paint_events
 246
 247   def _TimeAreaDict(self, paint_events, viewport):
 248     """Make a dict from time to adjusted area value for events at that time.
 249
 250     The adjusted area value of each paint event is determined by how many paint
 251     events cover the same rectangle, and whether it's a full-window paint event.
 252     "Adjusted area" can also be thought of as "points" of visual completeness --
 253     each rectangle has a certain number of points and these points are
 254     distributed amongst the paint events that paint that rectangle.
 255
 256     Args:
 257       paint_events: A list of paint events
 258       viewport: A tuple (width, height) of the window.
 259
 260     Returns:
 261       A dictionary of times of each paint event (in milliseconds) to the
 262       adjusted area that the paint event is worth.
 263     """
 264     width, height = viewport
 265     fullscreen_area = width * height
 266
 267     def ClippedArea(rectangle):
 268       """Returns rectangle area clipped to viewport size."""
 269       _, x0, y0, x1, y1 = rectangle
 270       clipped_width = max(0, min(width, x1) - max(0, x0))
 271       clipped_height = max(0, min(height, y1) - max(0, y0))
 272       return clipped_width * clipped_height
 273
 274     grouped = self._GroupEventByRectangle(paint_events)
 275     event_area_dict = collections.defaultdict(int)
 276
 277     for rectangle, events in grouped.items():
 278       # The area points for each rectangle are divided up among the paint
 279       # events in that rectangle.
 280       area = ClippedArea(rectangle)
 281       update_count = len(events)
 282       adjusted_area = float(area) / update_count
 283
 284       # Paint events for the largest-area rectangle are counted as 50%.
 285       if area == fullscreen_area:
 286         adjusted_area /= 2
 287
 288       for event in events:
 289         # The end time for an event is used for that event's time.
 290         event_time = event.end
 291         event_area_dict[event_time] += adjusted_area
 292
 293     return event_area_dict
 294
 295   def _GetRectangle(self, paint_event):
 296     """Get the specific rectangle on the screen for a paint event.
 297
 298     Each paint event belongs to a frame (as in html <frame> or <iframe>).
 299     This, together with location and dimensions, comprises a rectangle.
 300     In the WPT source, this 'rectangle' is also called a 'region'.
 301     """
 302     def GetBox(quad):
 303       """Gets top-left and bottom-right coordinates from paint event.
 304
 305       In the timeline data from devtools, paint rectangle dimensions are
 306       represented x-y coordinates of four corners, clockwise from the top-left.
 307       See: function WebInspector.TimelinePresentationModel.quadFromRectData
 308       in file src/out/Debug/obj/gen/devtools/TimelinePanel.js.
 309       """
 310       x0, y0, _, _, x1, y1, _, _ = quad
 311       return (x0, y0, x1, y1)
 312
 313     assert paint_event.name == 'Paint'
 314     frame = paint_event.args['frameId']
 315     return (frame,) + GetBox(paint_event.args['data']['clip'])
 316
 317   def _GroupEventByRectangle(self, paint_events):
 318     """Group all paint events according to the rectangle that they update."""
 319     result = collections.defaultdict(list)
 320     for event in paint_events:
 321       assert event.name == 'Paint'
 322       result[self._GetRectangle(event)].append(event)
 323     return result