tools/perf/benchmarks/peacekeeper.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """PeaceKeeper benchmark suite.
   6
   7 Peacekeeper measures browser's performance by testing its JavaScript
   8 functionality. JavaScript is a widely used programming language used in the
   9 creation of modern websites to provide features such as animation, navigation,
  10 forms and other common requirements. By measuring a browser's ability to handle
  11 commonly used JavaScript functions Peacekeeper can evaluate its performance.
  12 Peacekeeper scores are measured in operations per second or rendered frames per
  13 second depending on the test. Final Score is computed by calculating geometric
  14 mean of individual tests scores.
  15 """
  16
  17 from telemetry import benchmark
  18 from telemetry.page import page_set
  19 from telemetry.page import page_test
  20 from telemetry.util import statistics
  21 from telemetry.value import merge_values
  22 from telemetry.value import scalar
  23
  24
  25 class _PeaceKeeperMeasurement(page_test.PageTest):
  26
  27   def WillNavigateToPage(self, page, tab):
  28     page.script_to_evaluate_on_commit = """
  29         var __results = {};
  30         var _done = false;
  31         var __real_log = window.console.log;
  32         var test_frame = null;
  33         var benchmark = null;
  34         window.console.log = function(msg) {
  35           if (typeof(msg) == "string" && (msg.indexOf("benchmark")) == 0) {
  36             test_frame = document.getElementById("testFrame");
  37             benchmark =  test_frame.contentWindow.benchmark;
  38             test_frame.contentWindow.onbeforeunload = {};
  39             if ((msg.indexOf("Submit ok.")) != -1) {
  40               _done = true;
  41               __results["test"] = benchmark.testObjectName;
  42               __results["score"] = benchmark.test.result;
  43               if (typeof(benchmark.test.unit) != "undefined") {
  44                 __results["unit"] = benchmark.test.unit;
  45               } else {
  46                 __results["unit"] = benchmark.test.isFps ? "fps" : "ops";
  47               }
  48             }
  49           }
  50           __real_log.apply(this, [msg]);
  51         }
  52         """
  53
  54   def ValidateAndMeasurePage(self, _, tab, results):
  55     tab.WaitForJavaScriptExpression('_done', 600)
  56     result = tab.EvaluateJavaScript('__results')
  57
  58     results.AddValue(scalar.ScalarValue(
  59         results.current_page, '%s.Score' % result['test'], 'score',
  60         int(result['score'])), important=False)
  61
  62   def DidRunTest(self, browser, results):
  63     # Calculate geometric mean as the total for the combined tests.
  64     combined = merge_values.MergeLikeValuesFromDifferentPages(
  65         results.all_page_specific_values,
  66         group_by_name_suffix=True)
  67     combined_score = [x for x in combined if x.name == 'Score'][0]
  68     total = statistics.GeometricMean(combined_score.values)
  69     results.AddSummaryValue(
  70         scalar.ScalarValue(None, 'Total.Score', 'score', total))
  71
  72
  73 @benchmark.Disabled
  74 class PeaceKeeperBenchmark(benchmark.Benchmark):
  75   """A base class for Peackeeper benchmarks."""
  76   test = _PeaceKeeperMeasurement
  77
  78   def CreatePageSet(self, options):
  79     """Makes a PageSet for PeaceKeeper benchmarks."""
  80     # Subclasses are expected to define a class member called query_param.
  81     if not hasattr(self, 'test_param'):
  82       raise NotImplementedError('test_param not in PeaceKeeper benchmark.')
  83
  84     ps = page_set.PageSet(
  85       archive_data_file='../page_sets/data/peacekeeper_%s.json' % self.tag,
  86       make_javascript_deterministic=False)
  87     for test_name in self.test_param:
  88       ps.AddPageWithDefaultRunNavigate(
  89         ('http://peacekeeper.futuremark.com/run.action?debug=true&'
  90          'repeat=false&forceSuiteName=%s&forceTestName=%s') %
  91         (self.tag, test_name))
  92     return ps
  93
  94
  95 @benchmark.Disabled
  96 class PeaceKeeperRender(PeaceKeeperBenchmark):
  97   """PeaceKeeper rendering benchmark suite.
  98
  99   These tests measure your browser's ability to render and modify specific
 100   elements used in typical web pages. Rendering tests manipulate the DOM tree in
 101   real-time. The tests measure display updating speed (frames per seconds).
 102   """
 103   tag = 'render'
 104   test_param = ['renderGrid01',
 105                 'renderGrid02',
 106                 'renderGrid03',
 107                 'renderPhysics'
 108                ]
 109
 110
 111 @benchmark.Disabled
 112 class PeaceKeeperData(PeaceKeeperBenchmark):
 113   """PeaceKeeper Data operations benchmark suite.
 114
 115   These tests measure your browser's ability to add, remove and modify data
 116   stored in an array. The Data suite consists of two tests:
 117   1. arrayCombined: This test uses all features of the JavaScript Array object.
 118   This is a technical test that is not based on profiled data.
 119   The source data are different sized arrays of numbers.
 120   2. arrayWeighted: This test is similar to 'arrayCombined', but the load is
 121   balanced based on profiled data. The source data is a list of all the
 122   countries in the world.
 123   """
 124
 125   tag = 'array'
 126   test_param = ['arrayCombined01',
 127                 'arrayWeighted'
 128                ]
 129
 130
 131 @benchmark.Disabled
 132 class PeaceKeeperDom(PeaceKeeperBenchmark):
 133   """PeaceKeeper DOM operations benchmark suite.
 134
 135   These tests emulate the methods used to create typical dynamic webpages.
 136   The DOM tests are based on development experience and the capabilities of the
 137   jQuery framework.
 138   1. domGetElements: This test uses native DOM methods getElementById and
 139     getElementsByName. The elements are not modified.
 140   2. domDynamicCreationCreateElement: A common use of DOM is to dynamically
 141     create content with JavaScript, this test measures creating objects
 142     individually and then appending them to DOM.
 143   3. domDynamicCreationInnerHTML: This test is similarl to the previous one,
 144     but uses the innerHTML-method.
 145   4. domJQueryAttributeFilters: This test does a DOM query with jQuery.
 146     It searches elements with specific attributes.
 147   5. domJQueryBasicFilters: This test uses filters to query elements from DOM.
 148   6. domJQueryBasics: This test queries elements from DOM with basic methods.
 149     It is similar to domGetElements, but uses jQuery rather than native methods.
 150   7. domJQueryContentFilters: Query elements based on content. This does string
 151     searching and these methods are assumed to be time consuming.
 152   8. domJQueryHierarchy: Query elements based on hierarchy, such as getting
 153     sibling, parent or child nodes from a DOM tree.
 154   9. domQueryselector: QuerySelector, which allows JavaScript to search elements
 155     from the DOM tree directly without the need to iterate the whole tree
 156     through domGetElements.
 157   """
 158
 159   tag = 'dom'
 160   test_param = ['domGetElements',
 161                 'domDynamicCreationCreateElement',
 162                 'domDynamicCreationInnerHTML',
 163                 'domJQueryAttributeFilters',
 164                 'domJQueryBasicFilters',
 165                 'domJQueryBasics',
 166                 'domJQueryContentFilters',
 167                 'domJQueryHierarchy',
 168                 'domQueryselector'
 169                ]
 170
 171
 172 @benchmark.Disabled
 173 class PeaceKeeperTextParsing(PeaceKeeperBenchmark):
 174   """PeaceKeeper Text Parsing benchmark suite.
 175
 176   These tests measure your browser's performance in typical text manipulations
 177   such as using a profanity filter for chats, browser detection and form
 178   validation.
 179   1. stringChat: This test removes swearing from artificial chat messages.
 180     Test measures looping and string replace-method.
 181   2. stringDetectBrowser: This test uses string indexOf-method to detect browser
 182     and operating system.
 183   3. stringFilter: This test filters a list of movies with a given keyword.
 184     The behaviour is known as filtering select or continuous filter. It's used
 185     to give real time suggestions while a user is filling input fields.
 186     The test uses simple regular expressions.
 187   4. stringValidateForm: This test uses complex regular expressions to validate
 188     user input.
 189   5. stringWeighted: This is an artificial test. Methods used and their
 190     intensities are chosen based on profiled data.
 191   """
 192
 193   tag = 'string'
 194   test_param = ['stringChat',
 195                 'stringDetectBrowser',
 196                 'stringFilter',
 197                 'stringWeighted',
 198                 'stringValidateForm'
 199                ]
 200
 201
 202 @benchmark.Disabled
 203 class PeaceKeeperHTML5Canvas(PeaceKeeperBenchmark):
 204   """PeaceKeeper HTML5 Canvas benchmark suite.
 205
 206   These tests use HTML5 Canvas, which is a web technology for drawing and
 207   manipulating graphics without external plug-ins.
 208   1. experimentalRipple01: Simulates a 'water ripple' effect by using HTML 5
 209     Canvas. It measures the browser's ability to draw individual pixels.
 210   2. experimentalRipple02: Same test as 'experimentalRipple01', but with a
 211     larger canvas and thus a heavier workload.
 212   """
 213
 214   tag = 'experimental'
 215   test_param = ['experimentalRipple01',
 216                 'experimentalRipple02'
 217                ]
 218
 219
 220 @benchmark.Disabled
 221 class PeaceKeeperHTML5Capabilities(PeaceKeeperBenchmark):
 222   """PeaceKeeper HTML5 Capabilities benchmark suite.
 223
 224   These tests checks browser HTML5 capabilities support for WebGL, Video
 225   foramts, simple 2D sprite based games and web worker.
 226   This benchmark only tests HTML5 capability and thus is not calculate into the
 227   overall score.
 228   1. HTML5 - WebGL: WebGL allows full blown 3D graphics to be rendered in a
 229     browser without the need for any external plug-ins.
 230     a) webglSphere
 231   2. HTML5 - Video: hese tests find out which HTML5 video formats are supposed
 232     by your browser. Peacekeeper only checks if your browser is able to play a
 233     specific format, no other valuation is done.
 234     a) videoCodecH264
 235     b) videoCodecTheora
 236     c) videoCodecWebM
 237     d) videoPosterSupport
 238   3.HTML5 - Web Worker: These tests use HTML5 Web Worker, which allows
 239     JavaScript to multhread - ie. the ability to perform multiple actions
 240     concurrently.
 241     a) workerContrast01
 242     b) workerContrast02
 243   4. HTML5 - Game: This test simulates a simple 2D, sprite-based game.
 244     The test itself is the real game, and what is shown is a recorded play.
 245     a) gamingSpitfire
 246   """
 247
 248   tag = 'html5'
 249   test_param = ['webglSphere',
 250                 'gamingSpitfire',
 251                 'videoCodecH264',
 252                 'videoCodecTheora',
 253                 'videoCodecWebM',
 254                 'videoPosterSupport',
 255                 'workerContrast01',
 256                 'workerContrast02'
 257                 ]