tools/perf/measurements/smoothness_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import sys
   5
   6 from measurements import smoothness
   7 from metrics import power
   8 from telemetry import decorators
   9 from telemetry.core import exceptions
  10 from telemetry.core import wpr_modes
  11 from telemetry.page import page
  12 from telemetry.unittest_util import options_for_unittests
  13 from telemetry.unittest_util import page_test_test_case
  14
  15 class FakeTracingController(object):
  16   def __init__(self):
  17     self.category_filter = None
  18   def Start(self, _options, category_filter, _timeout):
  19     self.category_filter = category_filter
  20
  21 class FakePlatform(object):
  22   def __init__(self):
  23     self.tracing_controller = FakeTracingController()
  24   def CanMonitorPower(self):
  25     return False
  26
  27
  28 class FakeBrowser(object):
  29   def __init__(self):
  30     self.platform = FakePlatform()
  31
  32
  33 class AnimatedPage(page.Page):
  34   def __init__(self, page_set):
  35     super(AnimatedPage, self).__init__(
  36       url='file://animated_page.html',
  37       page_set=page_set, base_dir=page_set.base_dir)
  38
  39   def RunPageInteractions(self, action_runner):
  40     action_runner.Wait(.2)
  41
  42
  43 class FakeTab(object):
  44   def __init__(self):
  45     self.browser = FakeBrowser()
  46
  47   def ExecuteJavaScript(self, js):
  48     pass
  49
  50 class SmoothnessUnitTest(page_test_test_case.PageTestTestCase):
  51   """Smoke test for smoothness measurement
  52
  53      Runs smoothness measurement on a simple page and verifies
  54      that all metrics were added to the results. The test is purely functional,
  55      i.e. it only checks if the metrics are present and non-zero.
  56   """
  57   def testSyntheticDelayConfiguration(self):
  58     test_page = page.Page('http://dummy', None)
  59     test_page.synthetic_delays = {
  60         'cc.BeginMainFrame': { 'target_duration': 0.012 },
  61         'cc.DrawAndSwap': { 'target_duration': 0.012, 'mode': 'alternating' },
  62         'gpu.PresentingFrame': { 'target_duration': 0.012 }
  63     }
  64
  65     tab = FakeTab()
  66     measurement = smoothness.Smoothness()
  67     measurement.WillStartBrowser(tab.browser.platform)
  68     measurement.WillNavigateToPage(test_page, tab)
  69     measurement.WillRunActions(test_page, tab)
  70
  71     expected_category_filter = set([
  72         'DELAY(cc.BeginMainFrame;0.012000;static)',
  73         'DELAY(cc.DrawAndSwap;0.012000;alternating)',
  74         'DELAY(gpu.PresentingFrame;0.012000;static)',
  75         'benchmark'
  76     ])
  77     tracing_controller = tab.browser.platform.tracing_controller
  78     actual_category_filter = (
  79       tracing_controller.category_filter.included_categories)
  80
  81     # FIXME: Put blink.console into the expected above and remove these two
  82     # remove entries when the blink.console change has rolled into chromium.
  83     actual_category_filter.remove('webkit.console')
  84     actual_category_filter.remove('blink.console')
  85
  86     if expected_category_filter != actual_category_filter:
  87       sys.stderr.write("Expected category filter: %s\n" %
  88                        repr(expected_category_filter))
  89       sys.stderr.write("Actual category filter filter: %s\n" %
  90                        repr(actual_category_filter))
  91     self.assertEquals(expected_category_filter, actual_category_filter)
  92
  93   def setUp(self):
  94     self._options = options_for_unittests.GetCopy()
  95     self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
  96
  97   def testSmoothness(self):
  98     ps = self.CreatePageSetFromFileInUnittestDataDir('scrollable_page.html')
  99     measurement = smoothness.Smoothness()
 100     results = self.RunMeasurement(measurement, ps, options=self._options)
 101     self.assertEquals(0, len(results.failures))
 102
 103     frame_times = results.FindAllPageSpecificValuesNamed('frame_times')
 104     self.assertEquals(len(frame_times), 1)
 105     self.assertGreater(frame_times[0].GetRepresentativeNumber(), 0)
 106
 107     mean_frame_time = results.FindAllPageSpecificValuesNamed('mean_frame_time')
 108     self.assertEquals(len(mean_frame_time), 1)
 109     self.assertGreater(mean_frame_time[0].GetRepresentativeNumber(), 0)
 110
 111     frame_time_discrepancy = results.FindAllPageSpecificValuesNamed(
 112         'frame_time_discrepancy')
 113     self.assertEquals(len(frame_time_discrepancy), 1)
 114     self.assertGreater(frame_time_discrepancy[0].GetRepresentativeNumber(), 0)
 115
 116     percentage_smooth = results.FindAllPageSpecificValuesNamed(
 117         'percentage_smooth')
 118     self.assertEquals(len(percentage_smooth), 1)
 119     self.assertGreaterEqual(percentage_smooth[0].GetRepresentativeNumber(), 0)
 120
 121     mean_input_event_latency = results.FindAllPageSpecificValuesNamed(
 122         'mean_input_event_latency')
 123     if mean_input_event_latency:
 124       self.assertEquals(len(mean_input_event_latency), 1)
 125       self.assertGreater(
 126           mean_input_event_latency[0].GetRepresentativeNumber(), 0)
 127
 128   @decorators.Enabled('android')  # SurfaceFlinger is android-only
 129   def testSmoothnessSurfaceFlingerMetricsCalculated(self):
 130     ps = self.CreatePageSetFromFileInUnittestDataDir('scrollable_page.html')
 131     measurement = smoothness.Smoothness()
 132     results = self.RunMeasurement(measurement, ps, options=self._options)
 133     self.assertEquals(0, len(results.failures))
 134
 135     avg_surface_fps = results.FindAllPageSpecificValuesNamed('avg_surface_fps')
 136     self.assertEquals(1, len(avg_surface_fps))
 137     self.assertGreater(avg_surface_fps[0].GetRepresentativeNumber, 0)
 138
 139     jank_count = results.FindAllPageSpecificValuesNamed('jank_count')
 140     self.assertEquals(1, len(jank_count))
 141     self.assertGreater(jank_count[0].GetRepresentativeNumber(), -1)
 142
 143     max_frame_delay = results.FindAllPageSpecificValuesNamed('max_frame_delay')
 144     self.assertEquals(1, len(max_frame_delay))
 145     self.assertGreater(max_frame_delay[0].GetRepresentativeNumber, 0)
 146
 147     frame_lengths = results.FindAllPageSpecificValuesNamed('frame_lengths')
 148     self.assertEquals(1, len(frame_lengths))
 149     self.assertGreater(frame_lengths[0].GetRepresentativeNumber, 0)
 150
 151   @decorators.Disabled('mac', 'chromeos')  # http://crbug.com/403903
 152   def testSmoothnessForPageWithNoGesture(self):
 153     ps = self.CreateEmptyPageSet()
 154     ps.AddUserStory(AnimatedPage(ps))
 155
 156     measurement = smoothness.Smoothness()
 157     results = self.RunMeasurement(measurement, ps, options=self._options)
 158     self.assertEquals(0, len(results.failures))
 159
 160     percentage_smooth = results.FindAllPageSpecificValuesNamed(
 161         'percentage_smooth')
 162     self.assertEquals(len(percentage_smooth), 1)
 163     self.assertGreaterEqual(percentage_smooth[0].GetRepresentativeNumber(), 0)
 164
 165   def testCleanUpTrace(self):
 166     self.TestTracingCleanedUp(smoothness.Smoothness, self._options)
 167
 168   def testCleanUpPowerMetric(self):
 169     class FailPage(page.Page):
 170       def __init__(self, page_set):
 171         # pylint: disable=bad-super-call
 172         super(FailPage, self).__init__(
 173             url='file://blank.html',
 174             page_set=page_set, base_dir=page_set.base_dir)
 175       def RunPageInteractions(self, _):
 176         raise exceptions.IntentionalException
 177
 178     class FakePowerMetric(power.PowerMetric):
 179       start_called = False
 180       stop_called = True
 181       def Start(self, _1, _2):
 182         self.start_called = True
 183       def Stop(self, _1, _2):
 184         self.stop_called = True
 185
 186     ps = self.CreateEmptyPageSet()
 187     ps.AddUserStory(FailPage(ps))
 188
 189     class BuggyMeasurement(smoothness.Smoothness):
 190       fake_power = None
 191       # Inject fake power metric.
 192       def WillStartBrowser(self, platform):
 193         self.fake_power = self._power_metric = FakePowerMetric(platform)
 194
 195     measurement = BuggyMeasurement()
 196     try:
 197       self.RunMeasurement(measurement, ps)
 198     except exceptions.IntentionalException:
 199       pass
 200
 201     self.assertTrue(measurement.fake_power.start_called)
 202     self.assertTrue(measurement.fake_power.stop_called)