tools/perf/benchmarks/octane.py

   1 # Copyright 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Runs Octane 2.0 javascript benchmark.
   6
   7 Octane 2.0 is a modern benchmark that measures a JavaScript engine's performance
   8 by running a suite of tests representative of today's complex and demanding web
   9 applications. Octane's goal is to measure the performance of JavaScript code
  10 found in large, real-world web applications.
  11 Octane 2.0 consists of 17 tests, four more than Octane v1.
  12 """
  13
  14 import os
  15
  16 from telemetry import benchmark
  17 from telemetry import page as page_module
  18 from telemetry.page import page_set
  19 from telemetry.page import page_test
  20 from telemetry.util import statistics
  21 from telemetry.value import scalar
  22
  23 from metrics import power
  24
  25 _GB = 1024 * 1024 * 1024
  26
  27 DESCRIPTIONS = {
  28   'CodeLoad':
  29       'Measures how quickly a JavaScript engine can start executing code '
  30       'after loading a large JavaScript program, social widget being a common '
  31       'example. The source for test is derived from open source libraries '
  32       '(Closure, jQuery) (1,530 lines).',
  33   'Crypto':
  34       'Encryption and decryption benchmark based on code by Tom Wu '
  35       '(1698 lines).',
  36   'DeltaBlue':
  37       'One-way constraint solver, originally written in Smalltalk by John '
  38       'Maloney and Mario Wolczko (880 lines).',
  39   'EarleyBoyer':
  40       'Classic Scheme benchmarks, translated to JavaScript by Florian '
  41       'Loitsch\'s Scheme2Js compiler (4684 lines).',
  42   'Gameboy':
  43       'Emulate the portable console\'s architecture and runs a demanding 3D '
  44       'simulation, all in JavaScript (11,097 lines).',
  45   'Mandreel':
  46       'Runs the 3D Bullet Physics Engine ported from C++ to JavaScript via '
  47       'Mandreel (277,377 lines).',
  48   'NavierStokes':
  49       '2D NavierStokes equations solver, heavily manipulates double precision '
  50       'arrays. Based on Oliver Hunt\'s code (387 lines).',
  51   'PdfJS':
  52       'Mozilla\'s PDF Reader implemented in JavaScript. It measures decoding '
  53       'and interpretation time (33,056 lines).',
  54   'RayTrace':
  55       'Ray tracer benchmark based on code by Adam Burmister (904 lines).',
  56   'RegExp':
  57       'Regular expression benchmark generated by extracting regular '
  58       'expression operations from 50 of the most popular web pages '
  59       '(1761 lines).',
  60   'Richards':
  61       'OS kernel simulation benchmark, originally written in BCPL by Martin '
  62       'Richards (539 lines).',
  63   'Splay':
  64       'Data manipulation benchmark that deals with splay trees and exercises '
  65       'the automatic memory management subsystem (394 lines).',
  66 }
  67
  68
  69 class _OctaneMeasurement(page_test.PageTest):
  70   def __init__(self):
  71     super(_OctaneMeasurement, self).__init__()
  72     self._power_metric = None
  73
  74   def CustomizeBrowserOptions(self, options):
  75     power.PowerMetric.CustomizeBrowserOptions(options)
  76
  77   def WillStartBrowser(self, platform):
  78     self._power_metric = power.PowerMetric(platform)
  79
  80   def WillNavigateToPage(self, page, tab):
  81     memory_stats = tab.browser.memory_stats
  82     if ('SystemTotalPhysicalMemory' in memory_stats and
  83         memory_stats['SystemTotalPhysicalMemory'] < 1 * _GB):
  84       skipBenchmarks = '"zlib"'
  85     else:
  86       skipBenchmarks = ''
  87     page.script_to_evaluate_on_commit = """
  88         var __results = [];
  89         var __real_log = window.console.log;
  90         window.console.log = function(msg) {
  91           __results.push(msg);
  92           __real_log.apply(this, [msg]);
  93         }
  94         skipBenchmarks = [%s]
  95         """ % (skipBenchmarks)
  96
  97   def DidNavigateToPage(self, page, tab):
  98     self._power_metric.Start(page, tab)
  99
 100   def ValidateAndMeasurePage(self, page, tab, results):
 101     tab.WaitForJavaScriptExpression('window.completed', 10)
 102     tab.WaitForJavaScriptExpression(
 103         '!document.getElementById("progress-bar-container")', 1200)
 104
 105     self._power_metric.Stop(page, tab)
 106     self._power_metric.AddResults(tab, results)
 107
 108     results_log = tab.EvaluateJavaScript('__results')
 109     all_scores = []
 110     for output in results_log:
 111       # Split the results into score and test name.
 112       # results log e.g., "Richards: 18343"
 113       score_and_name = output.split(': ', 2)
 114       assert len(score_and_name) == 2, \
 115         'Unexpected result format "%s"' % score_and_name
 116       if 'Skipped' not in score_and_name[1]:
 117         name = score_and_name[0]
 118         score = int(score_and_name[1])
 119         results.AddValue(scalar.ScalarValue(
 120             results.current_page, name, 'score', score, important=False,
 121             description=DESCRIPTIONS.get(name)))
 122
 123         # Collect all test scores to compute geometric mean.
 124         all_scores.append(score)
 125     total = statistics.GeometricMean(all_scores)
 126     results.AddSummaryValue(
 127         scalar.ScalarValue(None, 'Total.Score', 'score', total,
 128                            description='Geometric mean of the scores of each '
 129                                        'individual benchmark in the Octane '
 130                                        'benchmark collection.'))
 131
 132
 133 class Octane(benchmark.Benchmark):
 134   """Google's Octane JavaScript benchmark.
 135
 136   http://octane-benchmark.googlecode.com/svn/latest/index.html
 137   """
 138   test = _OctaneMeasurement
 139
 140   @classmethod
 141   def Name(cls):
 142     return 'octane'
 143
 144   def CreatePageSet(self, options):
 145     ps = page_set.PageSet(
 146       archive_data_file='../page_sets/data/octane.json',
 147       file_path=os.path.abspath(__file__),
 148       bucket=page_set.PUBLIC_BUCKET)
 149     ps.AddUserStory(page_module.Page(
 150         'http://octane-benchmark.googlecode.com/svn/latest/index.html?auto=1',
 151         ps, ps.base_dir, make_javascript_deterministic=False))
 152     return ps