tools/perf/metrics/speedindex_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 # These tests access private methods in the speedindex module.
   6 # pylint: disable=W0212
   7
   8 import json
   9 import os
  10 import unittest
  11
  12 from telemetry.core import bitmap
  13 from telemetry.timeline import inspector_timeline_data
  14 from telemetry.timeline import model
  15 from metrics import speedindex
  16
  17 # Sample timeline data in the json format provided by devtools.
  18 # The sample events will be used in several tests below.
  19 _TEST_DIR = os.path.join(os.path.dirname(__file__), 'unittest_data')
  20 _SAMPLE_DATA = json.load(open(os.path.join(_TEST_DIR, 'sample_timeline.json')))
  21 _SAMPLE_TIMELINE_DATA = inspector_timeline_data.InspectorTimelineData(
  22     _SAMPLE_DATA)
  23 _SAMPLE_EVENTS = model.TimelineModel(
  24     timeline_data=_SAMPLE_TIMELINE_DATA).GetAllEvents()
  25
  26
  27 class FakeTimelineModel(object):
  28
  29   def __init__(self):
  30     self._events = []
  31
  32   def SetAllEvents(self, events):
  33     self._events = events
  34
  35   def GetAllEvents(self, recursive=True):
  36     assert recursive == True
  37     return self._events
  38
  39
  40 class FakeVideo(object):
  41
  42   def __init__(self, frames):
  43     self._frames = frames
  44
  45   def GetVideoFrameIter(self):
  46     for frame in self._frames:
  47       yield frame
  48
  49 class FakeBitmap(object):
  50
  51   def __init__(self, r, g, b):
  52     self._histogram = bitmap.ColorHistogram(r, g, b, bitmap.WHITE)
  53
  54   # pylint: disable=W0613
  55   def ColorHistogram(self, ignore_color=None, tolerance=None):
  56     return self._histogram
  57
  58
  59 class FakeTab(object):
  60
  61   def __init__(self, video_capture_result=None):
  62     self._timeline_model = FakeTimelineModel()
  63     self._javascript_result = None
  64     self._video_capture_result = FakeVideo(video_capture_result)
  65
  66   @property
  67   def timeline_model(self):
  68     return self._timeline_model
  69
  70   @property
  71   def video_capture_supported(self):
  72     return self._video_capture_result is not None
  73
  74   def SetEvaluateJavaScriptResult(self, result):
  75     self._javascript_result = result
  76
  77   def EvaluateJavaScript(self, _):
  78     return self._javascript_result
  79
  80   def StartVideoCapture(self, min_bitrate_mbps=1):
  81     assert self.video_capture_supported
  82     assert min_bitrate_mbps > 0
  83
  84   def StopVideoCapture(self):
  85     assert self.video_capture_supported
  86     return self._video_capture_result
  87
  88   def Highlight(self, _):
  89     pass
  90
  91
  92 class IncludedPaintEventsTest(unittest.TestCase):
  93   def testNumberPaintEvents(self):
  94     impl = speedindex.PaintRectSpeedIndexImpl()
  95     # In the sample data, there's one event that occurs before the layout event,
  96     # and one paint event that's not a leaf paint event.
  97     events = impl._IncludedPaintEvents(_SAMPLE_EVENTS)
  98     self.assertEqual(len(events), 5)
  99
 100
 101 class SpeedIndexImplTest(unittest.TestCase):
 102   def testAdjustedAreaDict(self):
 103     impl = speedindex.PaintRectSpeedIndexImpl()
 104     paint_events = impl._IncludedPaintEvents(_SAMPLE_EVENTS)
 105     viewport = 1000, 1000
 106     time_area_dict = impl._TimeAreaDict(paint_events, viewport)
 107     self.assertEqual(len(time_area_dict), 4)
 108     # The event that ends at time 100 is a fullscreen; it's discounted by half.
 109     self.assertEqual(time_area_dict[100], 500000)
 110     self.assertEqual(time_area_dict[300], 100000)
 111     self.assertEqual(time_area_dict[400], 200000)
 112     self.assertEqual(time_area_dict[800], 200000)
 113
 114   def testVideoCompleteness(self):
 115     frames = [
 116         (0.0, FakeBitmap([ 0, 0, 0,10], [ 0, 0, 0,10], [ 0, 0, 0,10])),
 117         (0.1, FakeBitmap([10, 0, 0, 0], [10, 0, 0, 0], [10, 0, 0, 0])),
 118         (0.2, FakeBitmap([ 0, 0, 2, 8], [ 0, 0, 4, 6], [ 0, 0, 1, 9])),
 119         (0.3, FakeBitmap([ 0, 3, 2, 5], [ 2, 1, 0, 7], [ 0, 3, 0, 7])),
 120         (0.4, FakeBitmap([ 0, 0, 1, 0], [ 0, 0, 1, 0], [ 0, 0, 1, 0])),
 121         (0.5, FakeBitmap([ 0, 4, 6, 0], [ 0, 4, 6, 0], [ 0, 4, 6, 0])),
 122     ]
 123     max_distance = 42.
 124
 125     tab = FakeTab(frames)
 126     impl = speedindex.VideoSpeedIndexImpl()
 127     impl.Start(tab)
 128     impl.Stop(tab)
 129     time_completeness = impl.GetTimeCompletenessList(tab)
 130     self.assertEqual(len(time_completeness), 6)
 131     self.assertEqual(time_completeness[0], (0.0, 0))
 132     self.assertTimeCompleteness(
 133         time_completeness[1], 0.1, 1 - (16 + 16 + 16) / max_distance)
 134     self.assertTimeCompleteness(
 135         time_completeness[2], 0.2, 1 - (12 + 10 + 13) / max_distance)
 136     self.assertTimeCompleteness(
 137         time_completeness[3], 0.3, 1 - (6 + 10 + 8) / max_distance)
 138     self.assertTimeCompleteness(
 139         time_completeness[4], 0.4, 1 - (4 + 4 + 4) / max_distance)
 140     self.assertEqual(time_completeness[5], (0.5, 1))
 141
 142   def testBlankPage(self):
 143     frames = [
 144         (0.0, FakeBitmap([0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1])),
 145         (0.1, FakeBitmap([0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1])),
 146         (0.2, FakeBitmap([1, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 1])),
 147         (0.3, FakeBitmap([0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1])),
 148     ]
 149     tab = FakeTab(frames)
 150     impl = speedindex.VideoSpeedIndexImpl()
 151     impl.Start(tab)
 152     impl.Stop(tab)
 153     time_completeness = impl.GetTimeCompletenessList(tab)
 154     self.assertEqual(len(time_completeness), 4)
 155     self.assertEqual(time_completeness[0], (0.0, 1.0))
 156     self.assertEqual(time_completeness[1], (0.1, 1.0))
 157     self.assertEqual(time_completeness[2], (0.2, 0.0))
 158     self.assertEqual(time_completeness[3], (0.3, 1.0))
 159
 160   def assertTimeCompleteness(self, time_completeness, time, completeness):
 161     self.assertEqual(time_completeness[0], time)
 162     self.assertAlmostEqual(time_completeness[1], completeness)
 163
 164
 165 class SpeedIndexTest(unittest.TestCase):
 166   def testWithSampleData(self):
 167     tab = FakeTab()
 168     impl = speedindex.PaintRectSpeedIndexImpl()
 169     viewport = 1000, 1000
 170     # Add up the parts of the speed index for each time interval.
 171     # Each part is the time interval multiplied by the proportion of the
 172     # total area value that is not yet painted for that interval.
 173     parts = []
 174     parts.append(100 * 1.0)
 175     parts.append(200 * 0.5)
 176     parts.append(100 * 0.4)
 177     parts.append(400 * 0.2)
 178     expected = sum(parts)  # 330.0
 179     tab.timeline_model.SetAllEvents(_SAMPLE_EVENTS)
 180     tab.SetEvaluateJavaScriptResult(viewport)
 181     actual = impl.CalculateSpeedIndex(tab)
 182     self.assertEqual(actual, expected)
 183
 184
 185 class WPTComparisonTest(unittest.TestCase):
 186   """Compare the speed index results with results given by webpagetest.org.
 187
 188   Given the same timeline data, both this speedindex metric and webpagetest.org
 189   should both return the same results. Fortunately, webpagetest.org also
 190   provides timeline data in json format along with the speed index results.
 191   """
 192
 193   def _TestJsonTimelineExpectation(self, filename, viewport, expected):
 194     """Check whether the result for some timeline data is as expected.
 195
 196     Args:
 197       filename: Filename of a json file which contains a
 198       expected: The result expected based on the WPT result.
 199     """
 200     tab = FakeTab()
 201     impl = speedindex.PaintRectSpeedIndexImpl()
 202     file_path = os.path.join(_TEST_DIR, filename)
 203     with open(file_path) as json_file:
 204       raw_events = json.load(json_file)
 205       timeline_data = inspector_timeline_data.InspectorTimelineData(raw_events)
 206       tab.timeline_model.SetAllEvents(
 207           model.TimelineModel(timeline_data=timeline_data).GetAllEvents())
 208       tab.SetEvaluateJavaScriptResult(viewport)
 209       actual = impl.CalculateSpeedIndex(tab)
 210       # The result might differ by 1 or more milliseconds due to rounding,
 211       # so compare to the nearest 10 milliseconds.
 212       self.assertAlmostEqual(actual, expected, places=-1)
 213
 214   def testCern(self):
 215     # Page: http://info.cern.ch/hypertext/WWW/TheProject.html
 216     # This page has only one paint event.
 217     self._TestJsonTimelineExpectation(
 218         'cern_repeat_timeline.json', (1014, 650), 379.0)
 219
 220   def testBaidu(self):
 221     # Page: http://www.baidu.com/
 222     # This page has several paint events, but no nested paint events.
 223     self._TestJsonTimelineExpectation(
 224         'baidu_repeat_timeline.json', (1014, 650), 1761.43)
 225
 226   def test2ch(self):
 227     # Page: http://2ch.net/
 228     # This page has several paint events, including nested paint events.
 229     self._TestJsonTimelineExpectation(
 230         '2ch_repeat_timeline.json', (997, 650), 674.58)
 231
 232
 233 if __name__ == "__main__":
 234   unittest.main()