tools/perf/benchmarks/blink_perf.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6
   7 from core import perf_benchmark
   8
   9 from telemetry import benchmark
  10 from telemetry.core import util
  11 from telemetry import page as page_module
  12 from telemetry.page import page_test
  13 from telemetry.page import shared_page_state
  14 from telemetry import story
  15 from telemetry.value import list_of_scalar_values
  16
  17 from page_sets import webgl_supported_shared_state
  18
  19
  20 BLINK_PERF_BASE_DIR = os.path.join(util.GetChromiumSrcDir(),
  21                                    'third_party', 'WebKit', 'PerformanceTests')
  22 SKIPPED_FILE = os.path.join(BLINK_PERF_BASE_DIR, 'Skipped')
  23
  24
  25 def CreateStorySetFromPath(path, skipped_file,
  26                           shared_page_state_class=(
  27                             shared_page_state.SharedPageState)):
  28   assert os.path.exists(path)
  29
  30   page_urls = []
  31   serving_dirs = set()
  32
  33   def _AddPage(path):
  34     if not path.endswith('.html'):
  35       return
  36     if '../' in open(path, 'r').read():
  37       # If the page looks like it references its parent dir, include it.
  38       serving_dirs.add(os.path.dirname(os.path.dirname(path)))
  39     page_urls.append('file://' + path.replace('\\', '/'))
  40
  41   def _AddDir(dir_path, skipped):
  42     for candidate_path in os.listdir(dir_path):
  43       if candidate_path == 'resources':
  44         continue
  45       candidate_path = os.path.join(dir_path, candidate_path)
  46       if candidate_path.startswith(skipped):
  47         continue
  48       if os.path.isdir(candidate_path):
  49         _AddDir(candidate_path, skipped)
  50       else:
  51         _AddPage(candidate_path)
  52
  53   if os.path.isdir(path):
  54     skipped = []
  55     if os.path.exists(skipped_file):
  56       for line in open(skipped_file, 'r').readlines():
  57         line = line.strip()
  58         if line and not line.startswith('#'):
  59           skipped_path = os.path.join(os.path.dirname(skipped_file), line)
  60           skipped.append(skipped_path.replace('/', os.sep))
  61     _AddDir(path, tuple(skipped))
  62   else:
  63     _AddPage(path)
  64   ps = story.StorySet(base_dir=os.getcwd()+os.sep,
  65                         serving_dirs=serving_dirs)
  66   for url in page_urls:
  67     ps.AddStory(page_module.Page(
  68       url, ps, ps.base_dir,
  69       shared_page_state_class=shared_page_state_class))
  70   return ps
  71
  72
  73 class _BlinkPerfMeasurement(page_test.PageTest):
  74   """Tuns a blink performance test and reports the results."""
  75   def __init__(self):
  76     super(_BlinkPerfMeasurement, self).__init__()
  77     with open(os.path.join(os.path.dirname(__file__),
  78                            'blink_perf.js'), 'r') as f:
  79       self._blink_perf_js = f.read()
  80
  81   def WillNavigateToPage(self, page, tab):
  82     page.script_to_evaluate_on_commit = self._blink_perf_js
  83
  84   def CustomizeBrowserOptions(self, options):
  85     options.AppendExtraBrowserArgs([
  86         '--js-flags=--expose_gc',
  87         '--enable-experimental-web-platform-features',
  88         '--disable-gesture-requirement-for-media-playback'
  89     ])
  90     if 'content-shell' in options.browser_type:
  91       options.AppendExtraBrowserArgs('--expose-internals-for-testing')
  92
  93   def ValidateAndMeasurePage(self, page, tab, results):
  94     tab.WaitForJavaScriptExpression('testRunner.isDone', 600)
  95
  96     log = tab.EvaluateJavaScript('document.getElementById("log").innerHTML')
  97
  98     for line in log.splitlines():
  99       if not line.startswith('values '):
 100         continue
 101       parts = line.split()
 102       values = [float(v.replace(',', '')) for v in parts[1:-1]]
 103       units = parts[-1]
 104       metric = page.display_name.split('.')[0].replace('/', '_')
 105       results.AddValue(list_of_scalar_values.ListOfScalarValues(
 106           results.current_page, metric, units, values))
 107
 108       break
 109
 110     print log
 111
 112
 113 class _BlinkPerfFullFrameMeasurement(_BlinkPerfMeasurement):
 114   def __init__(self):
 115     super(_BlinkPerfFullFrameMeasurement, self).__init__()
 116     self._blink_perf_js += '\nwindow.fullFrameMeasurement = true;'
 117
 118   def CustomizeBrowserOptions(self, options):
 119     super(_BlinkPerfFullFrameMeasurement, self).CustomizeBrowserOptions(
 120         options)
 121     # Full layout measurement needs content_shell with internals testing API.
 122     assert 'content-shell' in options.browser_type
 123     options.AppendExtraBrowserArgs(['--expose-internals-for-testing'])
 124
 125 @benchmark.Disabled  # http://crbug.com/500958
 126 class BlinkPerfBindings(perf_benchmark.PerfBenchmark):
 127   tag = 'bindings'
 128   test = _BlinkPerfMeasurement
 129
 130   @classmethod
 131   def Name(cls):
 132     return 'blink_perf.bindings'
 133
 134   def CreateStorySet(self, options):
 135     path = os.path.join(BLINK_PERF_BASE_DIR, 'Bindings')
 136     return CreateStorySetFromPath(path, SKIPPED_FILE)
 137
 138
 139 @benchmark.Enabled('content-shell')
 140 class BlinkPerfBlinkGC(perf_benchmark.PerfBenchmark):
 141   tag = 'blink_gc'
 142   test = _BlinkPerfMeasurement
 143
 144   @classmethod
 145   def Name(cls):
 146     return 'blink_perf.blink_gc'
 147
 148   def CreateStorySet(self, options):
 149     path = os.path.join(BLINK_PERF_BASE_DIR, 'BlinkGC')
 150     return CreateStorySetFromPath(path, SKIPPED_FILE)
 151
 152
 153 class BlinkPerfCSS(perf_benchmark.PerfBenchmark):
 154   tag = 'css'
 155   test = _BlinkPerfMeasurement
 156
 157   @classmethod
 158   def Name(cls):
 159     return 'blink_perf.css'
 160
 161   def CreateStorySet(self, options):
 162     path = os.path.join(BLINK_PERF_BASE_DIR, 'CSS')
 163     return CreateStorySetFromPath(path, SKIPPED_FILE)
 164
 165
 166 @benchmark.Disabled('xp',  # http://crbug.com/488059
 167                     'android')  # http://crbug.com/496707
 168 class BlinkPerfCanvas(perf_benchmark.PerfBenchmark):
 169   tag = 'canvas'
 170   test = _BlinkPerfMeasurement
 171
 172   @classmethod
 173   def Name(cls):
 174     return 'blink_perf.canvas'
 175
 176   def CreateStorySet(self, options):
 177     path = os.path.join(BLINK_PERF_BASE_DIR, 'Canvas')
 178     story_set = CreateStorySetFromPath(
 179       path, SKIPPED_FILE,
 180       shared_page_state_class=(
 181         webgl_supported_shared_state.WebGLSupportedSharedState))
 182     # WebGLSupportedSharedState requires the skipped_gpus property to
 183     # be set on each page.
 184     for page in story_set:
 185       page.skipped_gpus = []
 186     return story_set
 187
 188
 189 class BlinkPerfDOM(perf_benchmark.PerfBenchmark):
 190   tag = 'dom'
 191   test = _BlinkPerfMeasurement
 192
 193   @classmethod
 194   def Name(cls):
 195     return 'blink_perf.dom'
 196
 197   def CreateStorySet(self, options):
 198     path = os.path.join(BLINK_PERF_BASE_DIR, 'DOM')
 199     return CreateStorySetFromPath(path, SKIPPED_FILE)
 200
 201
 202 @benchmark.Disabled('release_x64')  # http://crbug.com/480999
 203 class BlinkPerfEvents(perf_benchmark.PerfBenchmark):
 204   tag = 'events'
 205   test = _BlinkPerfMeasurement
 206
 207   @classmethod
 208   def Name(cls):
 209     return 'blink_perf.events'
 210
 211   def CreateStorySet(self, options):
 212     path = os.path.join(BLINK_PERF_BASE_DIR, 'Events')
 213     return CreateStorySetFromPath(path, SKIPPED_FILE)
 214
 215
 216 @benchmark.Disabled('win8')  # http://crbug.com/462350
 217 class BlinkPerfLayout(perf_benchmark.PerfBenchmark):
 218   tag = 'layout'
 219   test = _BlinkPerfMeasurement
 220
 221   @classmethod
 222   def Name(cls):
 223     return 'blink_perf.layout'
 224
 225   def CreateStorySet(self, options):
 226     path = os.path.join(BLINK_PERF_BASE_DIR, 'Layout')
 227     return CreateStorySetFromPath(path, SKIPPED_FILE)
 228
 229
 230 @benchmark.Enabled('content-shell')
 231 class BlinkPerfLayoutFullLayout(BlinkPerfLayout):
 232   tag = 'layout_full_frame'
 233   test = _BlinkPerfFullFrameMeasurement
 234
 235   @classmethod
 236   def Name(cls):
 237     return 'blink_perf.layout_full_frame'
 238
 239
 240 class BlinkPerfMutation(perf_benchmark.PerfBenchmark):
 241   tag = 'mutation'
 242   test = _BlinkPerfMeasurement
 243
 244   @classmethod
 245   def Name(cls):
 246     return 'blink_perf.mutation'
 247
 248   def CreateStorySet(self, options):
 249     path = os.path.join(BLINK_PERF_BASE_DIR, 'Mutation')
 250     return CreateStorySetFromPath(path, SKIPPED_FILE)
 251
 252
 253 @benchmark.Disabled('win')  # crbug.com/488493
 254 class BlinkPerfParser(perf_benchmark.PerfBenchmark):
 255   tag = 'parser'
 256   test = _BlinkPerfMeasurement
 257
 258   @classmethod
 259   def Name(cls):
 260     return 'blink_perf.parser'
 261
 262   def CreateStorySet(self, options):
 263     path = os.path.join(BLINK_PERF_BASE_DIR, 'Parser')
 264     return CreateStorySetFromPath(path, SKIPPED_FILE)
 265
 266
 267 class BlinkPerfSVG(perf_benchmark.PerfBenchmark):
 268   tag = 'svg'
 269   test = _BlinkPerfMeasurement
 270
 271   @classmethod
 272   def Name(cls):
 273     return 'blink_perf.svg'
 274
 275   def CreateStorySet(self, options):
 276     path = os.path.join(BLINK_PERF_BASE_DIR, 'SVG')
 277     return CreateStorySetFromPath(path, SKIPPED_FILE)
 278
 279
 280 @benchmark.Enabled('content-shell')
 281 class BlinkPerfSVGFullLayout(BlinkPerfSVG):
 282   tag = 'svg_full_frame'
 283   test = _BlinkPerfFullFrameMeasurement
 284
 285   @classmethod
 286   def Name(cls):
 287     return 'blink_perf.svg_full_frame'
 288
 289
 290 class BlinkPerfShadowDOM(perf_benchmark.PerfBenchmark):
 291   tag = 'shadow_dom'
 292   test = _BlinkPerfMeasurement
 293
 294   @classmethod
 295   def Name(cls):
 296     return 'blink_perf.shadow_dom'
 297
 298   def CreateStorySet(self, options):
 299     path = os.path.join(BLINK_PERF_BASE_DIR, 'ShadowDOM')
 300     return CreateStorySetFromPath(path, SKIPPED_FILE)
 301
 302
 303 # This benchmark is for local testing, doesn't need to run on bots.
 304 @benchmark.Disabled()
 305 class BlinkPerfXMLHttpRequest(perf_benchmark.PerfBenchmark):
 306   tag = 'xml_http_request'
 307   test = _BlinkPerfMeasurement
 308
 309   @classmethod
 310   def Name(cls):
 311     return 'blink_perf.xml_http_request'
 312
 313   def CreateStorySet(self, options):
 314     path = os.path.join(BLINK_PERF_BASE_DIR, 'XMLHttpRequest')
 315     return CreateStorySetFromPath(path, SKIPPED_FILE)