tools/perf/benchmarks/blink_perf.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6
   7 from core import perf_benchmark
   8
   9 from telemetry import benchmark
  10 from telemetry.core import util
  11 from telemetry import page as page_module
  12 from telemetry.page import page_test
  13 from telemetry.page import shared_page_state
  14 from telemetry import story
  15 from telemetry.value import list_of_scalar_values
  16
  17 from benchmarks import pywebsocket_server
  18 from page_sets import webgl_supported_shared_state
  19
  20
  21 BLINK_PERF_BASE_DIR = os.path.join(util.GetChromiumSrcDir(),
  22                                    'third_party', 'WebKit', 'PerformanceTests')
  23 SKIPPED_FILE = os.path.join(BLINK_PERF_BASE_DIR, 'Skipped')
  24
  25
  26 def CreateStorySetFromPath(path, skipped_file,
  27                           shared_page_state_class=(
  28                             shared_page_state.SharedPageState)):
  29   assert os.path.exists(path)
  30
  31   page_urls = []
  32   serving_dirs = set()
  33
  34   def _AddPage(path):
  35     if not path.endswith('.html'):
  36       return
  37     if '../' in open(path, 'r').read():
  38       # If the page looks like it references its parent dir, include it.
  39       serving_dirs.add(os.path.dirname(os.path.dirname(path)))
  40     page_urls.append('file://' + path.replace('\\', '/'))
  41
  42   def _AddDir(dir_path, skipped):
  43     for candidate_path in os.listdir(dir_path):
  44       if candidate_path == 'resources':
  45         continue
  46       candidate_path = os.path.join(dir_path, candidate_path)
  47       if candidate_path.startswith(skipped):
  48         continue
  49       if os.path.isdir(candidate_path):
  50         _AddDir(candidate_path, skipped)
  51       else:
  52         _AddPage(candidate_path)
  53
  54   if os.path.isdir(path):
  55     skipped = []
  56     if os.path.exists(skipped_file):
  57       for line in open(skipped_file, 'r').readlines():
  58         line = line.strip()
  59         if line and not line.startswith('#'):
  60           skipped_path = os.path.join(os.path.dirname(skipped_file), line)
  61           skipped.append(skipped_path.replace('/', os.sep))
  62     _AddDir(path, tuple(skipped))
  63   else:
  64     _AddPage(path)
  65   ps = story.StorySet(base_dir=os.getcwd()+os.sep,
  66                         serving_dirs=serving_dirs)
  67   for url in page_urls:
  68     ps.AddStory(page_module.Page(
  69       url, ps, ps.base_dir,
  70       shared_page_state_class=shared_page_state_class))
  71   return ps
  72
  73
  74 class _BlinkPerfMeasurement(page_test.PageTest):
  75   """Tuns a blink performance test and reports the results."""
  76   def __init__(self):
  77     super(_BlinkPerfMeasurement, self).__init__()
  78     with open(os.path.join(os.path.dirname(__file__),
  79                            'blink_perf.js'), 'r') as f:
  80       self._blink_perf_js = f.read()
  81
  82   def WillNavigateToPage(self, page, tab):
  83     page.script_to_evaluate_on_commit = self._blink_perf_js
  84
  85   def CustomizeBrowserOptions(self, options):
  86     options.AppendExtraBrowserArgs([
  87         '--js-flags=--expose_gc',
  88         '--enable-experimental-web-platform-features',
  89         '--disable-gesture-requirement-for-media-playback'
  90     ])
  91     if 'content-shell' in options.browser_type:
  92       options.AppendExtraBrowserArgs('--expose-internals-for-testing')
  93
  94   def ValidateAndMeasurePage(self, page, tab, results):
  95     tab.WaitForJavaScriptExpression('testRunner.isDone', 600)
  96
  97     log = tab.EvaluateJavaScript('document.getElementById("log").innerHTML')
  98
  99     for line in log.splitlines():
 100       if not line.startswith('values '):
 101         continue
 102       parts = line.split()
 103       values = [float(v.replace(',', '')) for v in parts[1:-1]]
 104       units = parts[-1]
 105       metric = page.display_name.split('.')[0].replace('/', '_')
 106       results.AddValue(list_of_scalar_values.ListOfScalarValues(
 107           results.current_page, metric, units, values))
 108
 109       break
 110
 111     print log
 112
 113
 114 class _BlinkPerfFullFrameMeasurement(_BlinkPerfMeasurement):
 115   def __init__(self):
 116     super(_BlinkPerfFullFrameMeasurement, self).__init__()
 117     self._blink_perf_js += '\nwindow.fullFrameMeasurement = true;'
 118
 119   def CustomizeBrowserOptions(self, options):
 120     super(_BlinkPerfFullFrameMeasurement, self).CustomizeBrowserOptions(
 121         options)
 122     # Full layout measurement needs content_shell with internals testing API.
 123     assert 'content-shell' in options.browser_type
 124     options.AppendExtraBrowserArgs(['--expose-internals-for-testing'])
 125
 126
 127 class _BlinkPerfPywebsocketMeasurement(_BlinkPerfMeasurement):
 128   def CustomizeBrowserOptions(self, options):
 129     super(_BlinkPerfPywebsocketMeasurement, self).CustomizeBrowserOptions(
 130         options)
 131     # Cross-origin accesses are needed to run benchmarks spanning two servers,
 132     # the Telemetry's HTTP server and the pywebsocket server.
 133     options.AppendExtraBrowserArgs(['--disable-web-security'])
 134
 135
 136 class _SharedPywebsocketPageState(shared_page_state.SharedPageState):
 137   """Runs a pywebsocket server."""
 138   def __init__(self, test, finder_options, user_story_set):
 139     super(_SharedPywebsocketPageState, self).__init__(
 140         test, finder_options, user_story_set)
 141     self._pywebsocket_server = pywebsocket_server.PywebsocketServer()
 142     self.platform.StartLocalServer(self._pywebsocket_server)
 143
 144   def TearDownState(self):
 145     super(_SharedPywebsocketPageState, self).TearDownState()
 146     self._pywebsocket_server.Close()
 147
 148
 149 class BlinkPerfBindings(perf_benchmark.PerfBenchmark):
 150   tag = 'bindings'
 151   test = _BlinkPerfMeasurement
 152
 153   @classmethod
 154   def Name(cls):
 155     return 'blink_perf.bindings'
 156
 157   def CreateStorySet(self, options):
 158     path = os.path.join(BLINK_PERF_BASE_DIR, 'Bindings')
 159     return CreateStorySetFromPath(path, SKIPPED_FILE)
 160
 161
 162 @benchmark.Enabled('content-shell')
 163 class BlinkPerfBlinkGC(perf_benchmark.PerfBenchmark):
 164   tag = 'blink_gc'
 165   test = _BlinkPerfMeasurement
 166
 167   @classmethod
 168   def Name(cls):
 169     return 'blink_perf.blink_gc'
 170
 171   def CreateStorySet(self, options):
 172     path = os.path.join(BLINK_PERF_BASE_DIR, 'BlinkGC')
 173     return CreateStorySetFromPath(path, SKIPPED_FILE)
 174
 175
 176 class BlinkPerfCSS(perf_benchmark.PerfBenchmark):
 177   tag = 'css'
 178   test = _BlinkPerfMeasurement
 179
 180   @classmethod
 181   def Name(cls):
 182     return 'blink_perf.css'
 183
 184   def CreateStorySet(self, options):
 185     path = os.path.join(BLINK_PERF_BASE_DIR, 'CSS')
 186     return CreateStorySetFromPath(path, SKIPPED_FILE)
 187
 188
 189 @benchmark.Disabled('xp',  # http://crbug.com/488059
 190                     'android',  # http://crbug.com/496707
 191                     'reference')  # http://crbug.com/520092
 192 class BlinkPerfCanvas(perf_benchmark.PerfBenchmark):
 193   tag = 'canvas'
 194   test = _BlinkPerfMeasurement
 195
 196   @classmethod
 197   def Name(cls):
 198     return 'blink_perf.canvas'
 199
 200   def CreateStorySet(self, options):
 201     path = os.path.join(BLINK_PERF_BASE_DIR, 'Canvas')
 202     story_set = CreateStorySetFromPath(
 203       path, SKIPPED_FILE,
 204       shared_page_state_class=(
 205         webgl_supported_shared_state.WebGLSupportedSharedState))
 206     # WebGLSupportedSharedState requires the skipped_gpus property to
 207     # be set on each page.
 208     for page in story_set:
 209       page.skipped_gpus = []
 210     return story_set
 211
 212
 213 @benchmark.Disabled  # http://crbug.com/532093
 214 class BlinkPerfDOM(perf_benchmark.PerfBenchmark):
 215   tag = 'dom'
 216   test = _BlinkPerfMeasurement
 217
 218   @classmethod
 219   def Name(cls):
 220     return 'blink_perf.dom'
 221
 222   def CreateStorySet(self, options):
 223     path = os.path.join(BLINK_PERF_BASE_DIR, 'DOM')
 224     return CreateStorySetFromPath(path, SKIPPED_FILE)
 225
 226
 227 class BlinkPerfEvents(perf_benchmark.PerfBenchmark):
 228   tag = 'events'
 229   test = _BlinkPerfMeasurement
 230
 231   @classmethod
 232   def Name(cls):
 233     return 'blink_perf.events'
 234
 235   def CreateStorySet(self, options):
 236     path = os.path.join(BLINK_PERF_BASE_DIR, 'Events')
 237     return CreateStorySetFromPath(path, SKIPPED_FILE)
 238
 239
 240 @benchmark.Disabled('win8')  # http://crbug.com/462350
 241 class BlinkPerfLayout(perf_benchmark.PerfBenchmark):
 242   tag = 'layout'
 243   test = _BlinkPerfMeasurement
 244
 245   @classmethod
 246   def Name(cls):
 247     return 'blink_perf.layout'
 248
 249   def CreateStorySet(self, options):
 250     path = os.path.join(BLINK_PERF_BASE_DIR, 'Layout')
 251     return CreateStorySetFromPath(path, SKIPPED_FILE)
 252
 253
 254 @benchmark.Enabled('content-shell')
 255 class BlinkPerfLayoutFullLayout(BlinkPerfLayout):
 256   tag = 'layout_full_frame'
 257   test = _BlinkPerfFullFrameMeasurement
 258
 259   @classmethod
 260   def Name(cls):
 261     return 'blink_perf.layout_full_frame'
 262
 263
 264 @benchmark.Disabled('win',     # crbug.com/488493
 265                     'android') # crbug.com/527156
 266 class BlinkPerfParser(perf_benchmark.PerfBenchmark):
 267   tag = 'parser'
 268   test = _BlinkPerfMeasurement
 269
 270   @classmethod
 271   def Name(cls):
 272     return 'blink_perf.parser'
 273
 274   def CreateStorySet(self, options):
 275     path = os.path.join(BLINK_PERF_BASE_DIR, 'Parser')
 276     return CreateStorySetFromPath(path, SKIPPED_FILE)
 277
 278
 279 class BlinkPerfSVG(perf_benchmark.PerfBenchmark):
 280   tag = 'svg'
 281   test = _BlinkPerfMeasurement
 282
 283   @classmethod
 284   def Name(cls):
 285     return 'blink_perf.svg'
 286
 287   def CreateStorySet(self, options):
 288     path = os.path.join(BLINK_PERF_BASE_DIR, 'SVG')
 289     return CreateStorySetFromPath(path, SKIPPED_FILE)
 290
 291
 292 @benchmark.Enabled('content-shell')
 293 class BlinkPerfSVGFullLayout(BlinkPerfSVG):
 294   tag = 'svg_full_frame'
 295   test = _BlinkPerfFullFrameMeasurement
 296
 297   @classmethod
 298   def Name(cls):
 299     return 'blink_perf.svg_full_frame'
 300
 301
 302 class BlinkPerfShadowDOM(perf_benchmark.PerfBenchmark):
 303   tag = 'shadow_dom'
 304   test = _BlinkPerfMeasurement
 305
 306   @classmethod
 307   def Name(cls):
 308     return 'blink_perf.shadow_dom'
 309
 310   def CreateStorySet(self, options):
 311     path = os.path.join(BLINK_PERF_BASE_DIR, 'ShadowDOM')
 312     return CreateStorySetFromPath(path, SKIPPED_FILE)
 313
 314
 315 # This benchmark is for local testing, doesn't need to run on bots.
 316 @benchmark.Disabled()
 317 class BlinkPerfXMLHttpRequest(perf_benchmark.PerfBenchmark):
 318   tag = 'xml_http_request'
 319   test = _BlinkPerfMeasurement
 320
 321   @classmethod
 322   def Name(cls):
 323     return 'blink_perf.xml_http_request'
 324
 325   def CreateStorySet(self, options):
 326     path = os.path.join(BLINK_PERF_BASE_DIR, 'XMLHttpRequest')
 327     return CreateStorySetFromPath(path, SKIPPED_FILE)
 328
 329
 330 # Disabled on Windows and ChromeOS due to https://crbug.com/521887
 331 # Disabled on reference builds due to https://crbug.com/530374
 332 @benchmark.Disabled('win', 'chromeos', 'reference')
 333 class BlinkPerfPywebsocket(perf_benchmark.PerfBenchmark):
 334   tag = 'pywebsocket'
 335   test = _BlinkPerfPywebsocketMeasurement
 336
 337   @classmethod
 338   def Name(cls):
 339     return 'blink_perf.pywebsocket'
 340
 341   def CreateStorySet(self, options):
 342     path = os.path.join(BLINK_PERF_BASE_DIR, 'Pywebsocket')
 343     return CreateStorySetFromPath(path, SKIPPED_FILE,
 344         shared_page_state_class=_SharedPywebsocketPageState)