tools/perf/benchmarks/blink_perf.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6
   7 from telemetry import benchmark
   8 from telemetry.core import util
   9 from telemetry import page as page_module
  10 from telemetry.page import page_set
  11 from telemetry.page import page_test
  12 from telemetry.value import list_of_scalar_values
  13
  14
  15 BLINK_PERF_BASE_DIR = os.path.join(util.GetChromiumSrcDir(),
  16                                    'third_party', 'WebKit', 'PerformanceTests')
  17 SKIPPED_FILE = os.path.join(BLINK_PERF_BASE_DIR, 'Skipped')
  18
  19
  20 def CreatePageSetFromPath(path, skipped_file):
  21   assert os.path.exists(path)
  22
  23   page_urls = []
  24   serving_dirs = set()
  25
  26   def _AddPage(path):
  27     if not path.endswith('.html'):
  28       return
  29     if '../' in open(path, 'r').read():
  30       # If the page looks like it references its parent dir, include it.
  31       serving_dirs.add(os.path.dirname(os.path.dirname(path)))
  32     page_urls.append('file://' + path.replace('\\', '/'))
  33
  34   def _AddDir(dir_path, skipped):
  35     for candidate_path in os.listdir(dir_path):
  36       if candidate_path == 'resources':
  37         continue
  38       candidate_path = os.path.join(dir_path, candidate_path)
  39       if candidate_path.startswith(skipped):
  40         continue
  41       if os.path.isdir(candidate_path):
  42         _AddDir(candidate_path, skipped)
  43       else:
  44         _AddPage(candidate_path)
  45
  46   if os.path.isdir(path):
  47     skipped = []
  48     if os.path.exists(skipped_file):
  49       for line in open(skipped_file, 'r').readlines():
  50         line = line.strip()
  51         if line and not line.startswith('#'):
  52           skipped_path = os.path.join(os.path.dirname(skipped_file), line)
  53           skipped.append(skipped_path.replace('/', os.sep))
  54     _AddDir(path, tuple(skipped))
  55   else:
  56     _AddPage(path)
  57   ps = page_set.PageSet(file_path=os.getcwd()+os.sep,
  58                         serving_dirs=serving_dirs)
  59   for url in page_urls:
  60     ps.AddUserStory(page_module.Page(url, ps, ps.base_dir))
  61   return ps
  62
  63
  64 class _BlinkPerfMeasurement(page_test.PageTest):
  65   """Tuns a blink performance test and reports the results."""
  66   def __init__(self):
  67     super(_BlinkPerfMeasurement, self).__init__()
  68     with open(os.path.join(os.path.dirname(__file__),
  69                            'blink_perf.js'), 'r') as f:
  70       self._blink_perf_js = f.read()
  71
  72   def WillNavigateToPage(self, page, tab):
  73     page.script_to_evaluate_on_commit = self._blink_perf_js
  74
  75   def CustomizeBrowserOptions(self, options):
  76     options.AppendExtraBrowserArgs([
  77         '--js-flags=--expose_gc',
  78         '--enable-experimental-web-platform-features',
  79         '--disable-gesture-requirement-for-media-playback'
  80     ])
  81     if 'content-shell' in options.browser_type:
  82       options.AppendExtraBrowserArgs('--expose-internals-for-testing')
  83
  84   def ValidateAndMeasurePage(self, page, tab, results):
  85     tab.WaitForJavaScriptExpression('testRunner.isDone', 600)
  86
  87     log = tab.EvaluateJavaScript('document.getElementById("log").innerHTML')
  88
  89     for line in log.splitlines():
  90       if not line.startswith('values '):
  91         continue
  92       parts = line.split()
  93       values = [float(v.replace(',', '')) for v in parts[1:-1]]
  94       units = parts[-1]
  95       metric = page.display_name.split('.')[0].replace('/', '_')
  96       results.AddValue(list_of_scalar_values.ListOfScalarValues(
  97           results.current_page, metric, units, values))
  98
  99       break
 100
 101     print log
 102
 103
 104 class _BlinkPerfFullFrameMeasurement(_BlinkPerfMeasurement):
 105   def __init__(self):
 106     super(_BlinkPerfFullFrameMeasurement, self).__init__()
 107     self._blink_perf_js += '\nwindow.fullFrameMeasurement = true;'
 108
 109   def CustomizeBrowserOptions(self, options):
 110     super(_BlinkPerfFullFrameMeasurement, self).CustomizeBrowserOptions(
 111         options)
 112     # Full layout measurement needs content_shell with internals testing API.
 113     assert 'content-shell' in options.browser_type
 114     options.AppendExtraBrowserArgs(['--expose-internals-for-testing'])
 115
 116
 117 class BlinkPerfAnimation(benchmark.Benchmark):
 118   tag = 'animation'
 119   test = _BlinkPerfMeasurement
 120
 121   @classmethod
 122   def Name(cls):
 123     return 'blink_perf.animation'
 124
 125   def CreatePageSet(self, options):
 126     path = os.path.join(BLINK_PERF_BASE_DIR, 'Animation')
 127     return CreatePageSetFromPath(path, SKIPPED_FILE)
 128
 129
 130 class BlinkPerfBindings(benchmark.Benchmark):
 131   tag = 'bindings'
 132   test = _BlinkPerfMeasurement
 133
 134   @classmethod
 135   def Name(cls):
 136     return 'blink_perf.bindings'
 137
 138   def CreatePageSet(self, options):
 139     path = os.path.join(BLINK_PERF_BASE_DIR, 'Bindings')
 140     return CreatePageSetFromPath(path, SKIPPED_FILE)
 141
 142
 143 @benchmark.Enabled('content-shell')
 144 class BlinkPerfBlinkGC(benchmark.Benchmark):
 145   tag = 'blink_gc'
 146   test = _BlinkPerfMeasurement
 147
 148   @classmethod
 149   def Name(cls):
 150     return 'blink_perf.blink_gc'
 151
 152   def CreatePageSet(self, options):
 153     path = os.path.join(BLINK_PERF_BASE_DIR, 'BlinkGC')
 154     return CreatePageSetFromPath(path, SKIPPED_FILE)
 155
 156
 157 class BlinkPerfCSS(benchmark.Benchmark):
 158   tag = 'css'
 159   test = _BlinkPerfMeasurement
 160
 161   @classmethod
 162   def Name(cls):
 163     return 'blink_perf.css'
 164
 165   def CreatePageSet(self, options):
 166     path = os.path.join(BLINK_PERF_BASE_DIR, 'CSS')
 167     return CreatePageSetFromPath(path, SKIPPED_FILE)
 168
 169
 170 @benchmark.Disabled('win')  # crbug.com/455796
 171 class BlinkPerfCanvas(benchmark.Benchmark):
 172   tag = 'canvas'
 173   test = _BlinkPerfMeasurement
 174
 175   @classmethod
 176   def Name(cls):
 177     return 'blink_perf.canvas'
 178
 179   def CreatePageSet(self, options):
 180     path = os.path.join(BLINK_PERF_BASE_DIR, 'Canvas')
 181     return CreatePageSetFromPath(path, SKIPPED_FILE)
 182
 183
 184 class BlinkPerfDOM(benchmark.Benchmark):
 185   tag = 'dom'
 186   test = _BlinkPerfMeasurement
 187
 188   @classmethod
 189   def Name(cls):
 190     return 'blink_perf.dom'
 191
 192   def CreatePageSet(self, options):
 193     path = os.path.join(BLINK_PERF_BASE_DIR, 'DOM')
 194     return CreatePageSetFromPath(path, SKIPPED_FILE)
 195
 196
 197 class BlinkPerfEvents(benchmark.Benchmark):
 198   tag = 'events'
 199   test = _BlinkPerfMeasurement
 200
 201   @classmethod
 202   def Name(cls):
 203     return 'blink_perf.events'
 204
 205   def CreatePageSet(self, options):
 206     path = os.path.join(BLINK_PERF_BASE_DIR, 'Events')
 207     return CreatePageSetFromPath(path, SKIPPED_FILE)
 208
 209
 210 @benchmark.Disabled('win8')  # http://crbug.com/462350
 211 class BlinkPerfLayout(benchmark.Benchmark):
 212   tag = 'layout'
 213   test = _BlinkPerfMeasurement
 214
 215   @classmethod
 216   def Name(cls):
 217     return 'blink_perf.layout'
 218
 219   def CreatePageSet(self, options):
 220     path = os.path.join(BLINK_PERF_BASE_DIR, 'Layout')
 221     return CreatePageSetFromPath(path, SKIPPED_FILE)
 222
 223
 224 @benchmark.Enabled('content-shell')
 225 class BlinkPerfLayoutFullLayout(BlinkPerfLayout):
 226   tag = 'layout_full_frame'
 227   test = _BlinkPerfFullFrameMeasurement
 228
 229   @classmethod
 230   def Name(cls):
 231     return 'blink_perf.layout_full_frame'
 232
 233
 234 class BlinkPerfMutation(benchmark.Benchmark):
 235   tag = 'mutation'
 236   test = _BlinkPerfMeasurement
 237
 238   @classmethod
 239   def Name(cls):
 240     return 'blink_perf.mutation'
 241
 242   def CreatePageSet(self, options):
 243     path = os.path.join(BLINK_PERF_BASE_DIR, 'Mutation')
 244     return CreatePageSetFromPath(path, SKIPPED_FILE)
 245
 246
 247 class BlinkPerfParser(benchmark.Benchmark):
 248   tag = 'parser'
 249   test = _BlinkPerfMeasurement
 250
 251   @classmethod
 252   def Name(cls):
 253     return 'blink_perf.parser'
 254
 255   def CreatePageSet(self, options):
 256     path = os.path.join(BLINK_PERF_BASE_DIR, 'Parser')
 257     return CreatePageSetFromPath(path, SKIPPED_FILE)
 258
 259
 260 class BlinkPerfSVG(benchmark.Benchmark):
 261   tag = 'svg'
 262   test = _BlinkPerfMeasurement
 263
 264   @classmethod
 265   def Name(cls):
 266     return 'blink_perf.svg'
 267
 268   def CreatePageSet(self, options):
 269     path = os.path.join(BLINK_PERF_BASE_DIR, 'SVG')
 270     return CreatePageSetFromPath(path, SKIPPED_FILE)
 271
 272
 273 @benchmark.Enabled('content-shell')
 274 class BlinkPerfSVGFullLayout(BlinkPerfSVG):
 275   tag = 'svg_full_frame'
 276   test = _BlinkPerfFullFrameMeasurement
 277
 278   @classmethod
 279   def Name(cls):
 280     return 'blink_perf.svg_full_frame'
 281
 282
 283 class BlinkPerfShadowDOM(benchmark.Benchmark):
 284   tag = 'shadow_dom'
 285   test = _BlinkPerfMeasurement
 286
 287   @classmethod
 288   def Name(cls):
 289     return 'blink_perf.shadow_dom'
 290
 291   def CreatePageSet(self, options):
 292     path = os.path.join(BLINK_PERF_BASE_DIR, 'ShadowDOM')
 293     return CreatePageSetFromPath(path, SKIPPED_FILE)
 294
 295
 296 # This benchmark is for local testing, doesn't need to run on bots.
 297 @benchmark.Disabled()
 298 class BlinkPerfXMLHttpRequest(benchmark.Benchmark):
 299   tag = 'xml_http_request'
 300   test = _BlinkPerfMeasurement
 301
 302   @classmethod
 303   def Name(cls):
 304     return 'blink_perf.xml_http_request'
 305
 306   def CreatePageSet(self, options):
 307     path = os.path.join(BLINK_PERF_BASE_DIR, 'XMLHttpRequest')
 308     return CreatePageSetFromPath(path, SKIPPED_FILE)