tools/perf/benchmarks/blink_perf.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6
   7 from core import perf_benchmark
   8
   9 from telemetry import benchmark
  10 from telemetry.core import util
  11 from telemetry import page as page_module
  12 from telemetry.page import page_set
  13 from telemetry.page import page_test
  14 from telemetry.value import list_of_scalar_values
  15
  16
  17 BLINK_PERF_BASE_DIR = os.path.join(util.GetChromiumSrcDir(),
  18                                    'third_party', 'WebKit', 'PerformanceTests')
  19 SKIPPED_FILE = os.path.join(BLINK_PERF_BASE_DIR, 'Skipped')
  20
  21
  22 def CreatePageSetFromPath(path, skipped_file):
  23   assert os.path.exists(path)
  24
  25   page_urls = []
  26   serving_dirs = set()
  27
  28   def _AddPage(path):
  29     if not path.endswith('.html'):
  30       return
  31     if '../' in open(path, 'r').read():
  32       # If the page looks like it references its parent dir, include it.
  33       serving_dirs.add(os.path.dirname(os.path.dirname(path)))
  34     page_urls.append('file://' + path.replace('\\', '/'))
  35
  36   def _AddDir(dir_path, skipped):
  37     for candidate_path in os.listdir(dir_path):
  38       if candidate_path == 'resources':
  39         continue
  40       candidate_path = os.path.join(dir_path, candidate_path)
  41       if candidate_path.startswith(skipped):
  42         continue
  43       if os.path.isdir(candidate_path):
  44         _AddDir(candidate_path, skipped)
  45       else:
  46         _AddPage(candidate_path)
  47
  48   if os.path.isdir(path):
  49     skipped = []
  50     if os.path.exists(skipped_file):
  51       for line in open(skipped_file, 'r').readlines():
  52         line = line.strip()
  53         if line and not line.startswith('#'):
  54           skipped_path = os.path.join(os.path.dirname(skipped_file), line)
  55           skipped.append(skipped_path.replace('/', os.sep))
  56     _AddDir(path, tuple(skipped))
  57   else:
  58     _AddPage(path)
  59   ps = page_set.PageSet(file_path=os.getcwd()+os.sep,
  60                         serving_dirs=serving_dirs)
  61   for url in page_urls:
  62     ps.AddUserStory(page_module.Page(url, ps, ps.base_dir))
  63   return ps
  64
  65
  66 class _BlinkPerfMeasurement(page_test.PageTest):
  67   """Tuns a blink performance test and reports the results."""
  68   def __init__(self):
  69     super(_BlinkPerfMeasurement, self).__init__()
  70     with open(os.path.join(os.path.dirname(__file__),
  71                            'blink_perf.js'), 'r') as f:
  72       self._blink_perf_js = f.read()
  73
  74   def WillNavigateToPage(self, page, tab):
  75     page.script_to_evaluate_on_commit = self._blink_perf_js
  76
  77   def CustomizeBrowserOptions(self, options):
  78     options.AppendExtraBrowserArgs([
  79         '--js-flags=--expose_gc',
  80         '--enable-experimental-web-platform-features',
  81         '--disable-gesture-requirement-for-media-playback'
  82     ])
  83     if 'content-shell' in options.browser_type:
  84       options.AppendExtraBrowserArgs('--expose-internals-for-testing')
  85
  86   def ValidateAndMeasurePage(self, page, tab, results):
  87     tab.WaitForJavaScriptExpression('testRunner.isDone', 600)
  88
  89     log = tab.EvaluateJavaScript('document.getElementById("log").innerHTML')
  90
  91     for line in log.splitlines():
  92       if not line.startswith('values '):
  93         continue
  94       parts = line.split()
  95       values = [float(v.replace(',', '')) for v in parts[1:-1]]
  96       units = parts[-1]
  97       metric = page.display_name.split('.')[0].replace('/', '_')
  98       results.AddValue(list_of_scalar_values.ListOfScalarValues(
  99           results.current_page, metric, units, values))
 100
 101       break
 102
 103     print log
 104
 105
 106 class _BlinkPerfFullFrameMeasurement(_BlinkPerfMeasurement):
 107   def __init__(self):
 108     super(_BlinkPerfFullFrameMeasurement, self).__init__()
 109     self._blink_perf_js += '\nwindow.fullFrameMeasurement = true;'
 110
 111   def CustomizeBrowserOptions(self, options):
 112     super(_BlinkPerfFullFrameMeasurement, self).CustomizeBrowserOptions(
 113         options)
 114     # Full layout measurement needs content_shell with internals testing API.
 115     assert 'content-shell' in options.browser_type
 116     options.AppendExtraBrowserArgs(['--expose-internals-for-testing'])
 117
 118
 119 class BlinkPerfAnimation(perf_benchmark.PerfBenchmark):
 120   tag = 'animation'
 121   test = _BlinkPerfMeasurement
 122
 123   @classmethod
 124   def Name(cls):
 125     return 'blink_perf.animation'
 126
 127   def CreatePageSet(self, options):
 128     path = os.path.join(BLINK_PERF_BASE_DIR, 'Animation')
 129     return CreatePageSetFromPath(path, SKIPPED_FILE)
 130
 131
 132 class BlinkPerfBindings(perf_benchmark.PerfBenchmark):
 133   tag = 'bindings'
 134   test = _BlinkPerfMeasurement
 135
 136   @classmethod
 137   def Name(cls):
 138     return 'blink_perf.bindings'
 139
 140   def CreatePageSet(self, options):
 141     path = os.path.join(BLINK_PERF_BASE_DIR, 'Bindings')
 142     return CreatePageSetFromPath(path, SKIPPED_FILE)
 143
 144
 145 @benchmark.Enabled('content-shell')
 146 class BlinkPerfBlinkGC(perf_benchmark.PerfBenchmark):
 147   tag = 'blink_gc'
 148   test = _BlinkPerfMeasurement
 149
 150   @classmethod
 151   def Name(cls):
 152     return 'blink_perf.blink_gc'
 153
 154   def CreatePageSet(self, options):
 155     path = os.path.join(BLINK_PERF_BASE_DIR, 'BlinkGC')
 156     return CreatePageSetFromPath(path, SKIPPED_FILE)
 157
 158
 159 class BlinkPerfCSS(perf_benchmark.PerfBenchmark):
 160   tag = 'css'
 161   test = _BlinkPerfMeasurement
 162
 163   @classmethod
 164   def Name(cls):
 165     return 'blink_perf.css'
 166
 167   def CreatePageSet(self, options):
 168     path = os.path.join(BLINK_PERF_BASE_DIR, 'CSS')
 169     return CreatePageSetFromPath(path, SKIPPED_FILE)
 170
 171
 172 @benchmark.Disabled('linux', # http://crbug.com/488059
 173                     'xp')  # http://crbug.com/488059
 174 class BlinkPerfCanvas(perf_benchmark.PerfBenchmark):
 175   tag = 'canvas'
 176   test = _BlinkPerfMeasurement
 177
 178   @classmethod
 179   def Name(cls):
 180     return 'blink_perf.canvas'
 181
 182   def CreatePageSet(self, options):
 183     path = os.path.join(BLINK_PERF_BASE_DIR, 'Canvas')
 184     return CreatePageSetFromPath(path, SKIPPED_FILE)
 185
 186
 187 class BlinkPerfDOM(perf_benchmark.PerfBenchmark):
 188   tag = 'dom'
 189   test = _BlinkPerfMeasurement
 190
 191   @classmethod
 192   def Name(cls):
 193     return 'blink_perf.dom'
 194
 195   def CreatePageSet(self, options):
 196     path = os.path.join(BLINK_PERF_BASE_DIR, 'DOM')
 197     return CreatePageSetFromPath(path, SKIPPED_FILE)
 198
 199
 200 @benchmark.Disabled('release_x64')  # http://crbug.com/480999
 201 class BlinkPerfEvents(perf_benchmark.PerfBenchmark):
 202   tag = 'events'
 203   test = _BlinkPerfMeasurement
 204
 205   @classmethod
 206   def Name(cls):
 207     return 'blink_perf.events'
 208
 209   def CreatePageSet(self, options):
 210     path = os.path.join(BLINK_PERF_BASE_DIR, 'Events')
 211     return CreatePageSetFromPath(path, SKIPPED_FILE)
 212
 213
 214 @benchmark.Disabled('win8')  # http://crbug.com/462350
 215 class BlinkPerfLayout(perf_benchmark.PerfBenchmark):
 216   tag = 'layout'
 217   test = _BlinkPerfMeasurement
 218
 219   @classmethod
 220   def Name(cls):
 221     return 'blink_perf.layout'
 222
 223   def CreatePageSet(self, options):
 224     path = os.path.join(BLINK_PERF_BASE_DIR, 'Layout')
 225     return CreatePageSetFromPath(path, SKIPPED_FILE)
 226
 227
 228 @benchmark.Enabled('content-shell')
 229 class BlinkPerfLayoutFullLayout(BlinkPerfLayout):
 230   tag = 'layout_full_frame'
 231   test = _BlinkPerfFullFrameMeasurement
 232
 233   @classmethod
 234   def Name(cls):
 235     return 'blink_perf.layout_full_frame'
 236
 237
 238 class BlinkPerfMutation(perf_benchmark.PerfBenchmark):
 239   tag = 'mutation'
 240   test = _BlinkPerfMeasurement
 241
 242   @classmethod
 243   def Name(cls):
 244     return 'blink_perf.mutation'
 245
 246   def CreatePageSet(self, options):
 247     path = os.path.join(BLINK_PERF_BASE_DIR, 'Mutation')
 248     return CreatePageSetFromPath(path, SKIPPED_FILE)
 249
 250
 251 @benchmark.Disabled('win')  # crbug.com/488493
 252 class BlinkPerfParser(perf_benchmark.PerfBenchmark):
 253   tag = 'parser'
 254   test = _BlinkPerfMeasurement
 255
 256   @classmethod
 257   def Name(cls):
 258     return 'blink_perf.parser'
 259
 260   def CreatePageSet(self, options):
 261     path = os.path.join(BLINK_PERF_BASE_DIR, 'Parser')
 262     return CreatePageSetFromPath(path, SKIPPED_FILE)
 263
 264
 265 class BlinkPerfSVG(perf_benchmark.PerfBenchmark):
 266   tag = 'svg'
 267   test = _BlinkPerfMeasurement
 268
 269   @classmethod
 270   def Name(cls):
 271     return 'blink_perf.svg'
 272
 273   def CreatePageSet(self, options):
 274     path = os.path.join(BLINK_PERF_BASE_DIR, 'SVG')
 275     return CreatePageSetFromPath(path, SKIPPED_FILE)
 276
 277
 278 @benchmark.Enabled('content-shell')
 279 class BlinkPerfSVGFullLayout(BlinkPerfSVG):
 280   tag = 'svg_full_frame'
 281   test = _BlinkPerfFullFrameMeasurement
 282
 283   @classmethod
 284   def Name(cls):
 285     return 'blink_perf.svg_full_frame'
 286
 287
 288 class BlinkPerfShadowDOM(perf_benchmark.PerfBenchmark):
 289   tag = 'shadow_dom'
 290   test = _BlinkPerfMeasurement
 291
 292   @classmethod
 293   def Name(cls):
 294     return 'blink_perf.shadow_dom'
 295
 296   def CreatePageSet(self, options):
 297     path = os.path.join(BLINK_PERF_BASE_DIR, 'ShadowDOM')
 298     return CreatePageSetFromPath(path, SKIPPED_FILE)
 299
 300
 301 # This benchmark is for local testing, doesn't need to run on bots.
 302 @benchmark.Disabled()
 303 class BlinkPerfXMLHttpRequest(perf_benchmark.PerfBenchmark):
 304   tag = 'xml_http_request'
 305   test = _BlinkPerfMeasurement
 306
 307   @classmethod
 308   def Name(cls):
 309     return 'blink_perf.xml_http_request'
 310
 311   def CreatePageSet(self, options):
 312     path = os.path.join(BLINK_PERF_BASE_DIR, 'XMLHttpRequest')
 313     return CreatePageSetFromPath(path, SKIPPED_FILE)