tools/perf/measurements/smoothness_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import sys
   5
   6 from telemetry.core import exceptions
   7 from telemetry.core import wpr_modes
   8 from telemetry import decorators
   9 from telemetry.page import page
  10 from telemetry.unittest_util import options_for_unittests
  11 from telemetry.unittest_util import page_test_test_case
  12
  13 from measurements import smoothness
  14 from metrics import power
  15
  16
  17 class FakeTracingController(object):
  18   def __init__(self):
  19     self.category_filter = None
  20   def Start(self, _options, category_filter, _timeout):
  21     self.category_filter = category_filter
  22
  23 class FakePlatform(object):
  24   def __init__(self):
  25     self.tracing_controller = FakeTracingController()
  26   def CanMonitorPower(self):
  27     return False
  28
  29
  30 class FakeBrowser(object):
  31   def __init__(self):
  32     self.platform = FakePlatform()
  33
  34
  35 class AnimatedPage(page.Page):
  36   def __init__(self, page_set):
  37     super(AnimatedPage, self).__init__(
  38       url='file://animated_page.html',
  39       page_set=page_set, base_dir=page_set.base_dir)
  40
  41   def RunPageInteractions(self, action_runner):
  42     action_runner.Wait(.2)
  43
  44
  45 class FakeTab(object):
  46   def __init__(self):
  47     self.browser = FakeBrowser()
  48
  49   def ExecuteJavaScript(self, js):
  50     pass
  51
  52 class SmoothnessUnitTest(page_test_test_case.PageTestTestCase):
  53   """Smoke test for smoothness measurement
  54
  55      Runs smoothness measurement on a simple page and verifies
  56      that all metrics were added to the results. The test is purely functional,
  57      i.e. it only checks if the metrics are present and non-zero.
  58   """
  59   def testSyntheticDelayConfiguration(self):
  60     test_page = page.Page('http://dummy', None)
  61     test_page.synthetic_delays = {
  62         'cc.BeginMainFrame': { 'target_duration': 0.012 },
  63         'cc.DrawAndSwap': { 'target_duration': 0.012, 'mode': 'alternating' },
  64         'gpu.PresentingFrame': { 'target_duration': 0.012 }
  65     }
  66
  67     tab = FakeTab()
  68     measurement = smoothness.Smoothness()
  69     measurement.WillStartBrowser(tab.browser.platform)
  70     measurement.WillNavigateToPage(test_page, tab)
  71     measurement.WillRunActions(test_page, tab)
  72
  73     expected_category_filter = set([
  74         'DELAY(cc.BeginMainFrame;0.012000;static)',
  75         'DELAY(cc.DrawAndSwap;0.012000;alternating)',
  76         'DELAY(gpu.PresentingFrame;0.012000;static)',
  77         'benchmark'
  78     ])
  79     tracing_controller = tab.browser.platform.tracing_controller
  80     actual_category_filter = (
  81       tracing_controller.category_filter.included_categories)
  82
  83     # FIXME: Put blink.console into the expected above and remove these two
  84     # remove entries when the blink.console change has rolled into chromium.
  85     actual_category_filter.remove('webkit.console')
  86     actual_category_filter.remove('blink.console')
  87
  88     if expected_category_filter != actual_category_filter:
  89       sys.stderr.write("Expected category filter: %s\n" %
  90                        repr(expected_category_filter))
  91       sys.stderr.write("Actual category filter filter: %s\n" %
  92                        repr(actual_category_filter))
  93     self.assertEquals(expected_category_filter, actual_category_filter)
  94
  95   def setUp(self):
  96     self._options = options_for_unittests.GetCopy()
  97     self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
  98
  99   def testSmoothness(self):
 100     ps = self.CreatePageSetFromFileInUnittestDataDir('scrollable_page.html')
 101     measurement = smoothness.Smoothness()
 102     results = self.RunMeasurement(measurement, ps, options=self._options)
 103     self.assertEquals(0, len(results.failures))
 104
 105     frame_times = results.FindAllPageSpecificValuesNamed('frame_times')
 106     self.assertEquals(len(frame_times), 1)
 107     self.assertGreater(frame_times[0].GetRepresentativeNumber(), 0)
 108
 109     mean_frame_time = results.FindAllPageSpecificValuesNamed('mean_frame_time')
 110     self.assertEquals(len(mean_frame_time), 1)
 111     self.assertGreater(mean_frame_time[0].GetRepresentativeNumber(), 0)
 112
 113     frame_time_discrepancy = results.FindAllPageSpecificValuesNamed(
 114         'frame_time_discrepancy')
 115     self.assertEquals(len(frame_time_discrepancy), 1)
 116     self.assertGreater(frame_time_discrepancy[0].GetRepresentativeNumber(), 0)
 117
 118     percentage_smooth = results.FindAllPageSpecificValuesNamed(
 119         'percentage_smooth')
 120     self.assertEquals(len(percentage_smooth), 1)
 121     self.assertGreaterEqual(percentage_smooth[0].GetRepresentativeNumber(), 0)
 122
 123     mean_input_event_latency = results.FindAllPageSpecificValuesNamed(
 124         'mean_input_event_latency')
 125     if mean_input_event_latency:
 126       self.assertEquals(len(mean_input_event_latency), 1)
 127       self.assertGreater(
 128           mean_input_event_latency[0].GetRepresentativeNumber(), 0)
 129
 130   @decorators.Enabled('android')  # SurfaceFlinger is android-only
 131   def testSmoothnessSurfaceFlingerMetricsCalculated(self):
 132     ps = self.CreatePageSetFromFileInUnittestDataDir('scrollable_page.html')
 133     measurement = smoothness.Smoothness()
 134     results = self.RunMeasurement(measurement, ps, options=self._options)
 135     self.assertEquals(0, len(results.failures))
 136
 137     avg_surface_fps = results.FindAllPageSpecificValuesNamed('avg_surface_fps')
 138     self.assertEquals(1, len(avg_surface_fps))
 139     self.assertGreater(avg_surface_fps[0].GetRepresentativeNumber, 0)
 140
 141     jank_count = results.FindAllPageSpecificValuesNamed('jank_count')
 142     self.assertEquals(1, len(jank_count))
 143     self.assertGreater(jank_count[0].GetRepresentativeNumber(), -1)
 144
 145     max_frame_delay = results.FindAllPageSpecificValuesNamed('max_frame_delay')
 146     self.assertEquals(1, len(max_frame_delay))
 147     self.assertGreater(max_frame_delay[0].GetRepresentativeNumber, 0)
 148
 149     frame_lengths = results.FindAllPageSpecificValuesNamed('frame_lengths')
 150     self.assertEquals(1, len(frame_lengths))
 151     self.assertGreater(frame_lengths[0].GetRepresentativeNumber, 0)
 152
 153   @decorators.Disabled('mac', 'chromeos')  # http://crbug.com/403903
 154   def testSmoothnessForPageWithNoGesture(self):
 155     ps = self.CreateEmptyPageSet()
 156     ps.AddUserStory(AnimatedPage(ps))
 157
 158     measurement = smoothness.Smoothness()
 159     results = self.RunMeasurement(measurement, ps, options=self._options)
 160     self.assertEquals(0, len(results.failures))
 161
 162     percentage_smooth = results.FindAllPageSpecificValuesNamed(
 163         'percentage_smooth')
 164     self.assertEquals(len(percentage_smooth), 1)
 165     self.assertGreaterEqual(percentage_smooth[0].GetRepresentativeNumber(), 0)
 166
 167   def testCleanUpTrace(self):
 168     self.TestTracingCleanedUp(smoothness.Smoothness, self._options)
 169
 170   def testCleanUpPowerMetric(self):
 171     class FailPage(page.Page):
 172       def __init__(self, page_set):
 173         # pylint: disable=bad-super-call
 174         super(FailPage, self).__init__(
 175             url='file://blank.html',
 176             page_set=page_set, base_dir=page_set.base_dir)
 177       def RunPageInteractions(self, _):
 178         raise exceptions.IntentionalException
 179
 180     class FakePowerMetric(power.PowerMetric):
 181       start_called = False
 182       stop_called = True
 183       def Start(self, _1, _2):
 184         self.start_called = True
 185       def Stop(self, _1, _2):
 186         self.stop_called = True
 187
 188     ps = self.CreateEmptyPageSet()
 189     ps.AddUserStory(FailPage(ps))
 190
 191     class BuggyMeasurement(smoothness.Smoothness):
 192       fake_power = None
 193       # Inject fake power metric.
 194       def WillStartBrowser(self, platform):
 195         self.fake_power = self._power_metric = FakePowerMetric(platform)
 196
 197     measurement = BuggyMeasurement()
 198     try:
 199       self.RunMeasurement(measurement, ps)
 200     except exceptions.IntentionalException:
 201       pass
 202
 203     self.assertTrue(measurement.fake_power.start_called)
 204     self.assertTrue(measurement.fake_power.stop_called)