tools/perf/benchmarks/octane.py

   1 # Copyright 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Runs Octane 2.0 javascript benchmark.
   6
   7 Octane 2.0 is a modern benchmark that measures a JavaScript engine's performance
   8 by running a suite of tests representative of today's complex and demanding web
   9 applications. Octane's goal is to measure the performance of JavaScript code
  10 found in large, real-world web applications.
  11 Octane 2.0 consists of 17 tests, four more than Octane v1.
  12 """
  13
  14 import os
  15
  16 from core import perf_benchmark
  17
  18 from telemetry import page as page_module
  19 from telemetry.page import page_test
  20 from telemetry import story
  21 from telemetry.util import statistics
  22 from telemetry.value import scalar
  23
  24 from metrics import power
  25
  26 _GB = 1024 * 1024 * 1024
  27
  28 DESCRIPTIONS = {
  29   'CodeLoad':
  30       'Measures how quickly a JavaScript engine can start executing code '
  31       'after loading a large JavaScript program, social widget being a common '
  32       'example. The source for test is derived from open source libraries '
  33       '(Closure, jQuery) (1,530 lines).',
  34   'Crypto':
  35       'Encryption and decryption benchmark based on code by Tom Wu '
  36       '(1698 lines).',
  37   'DeltaBlue':
  38       'One-way constraint solver, originally written in Smalltalk by John '
  39       'Maloney and Mario Wolczko (880 lines).',
  40   'EarleyBoyer':
  41       'Classic Scheme benchmarks, translated to JavaScript by Florian '
  42       'Loitsch\'s Scheme2Js compiler (4684 lines).',
  43   'Gameboy':
  44       'Emulate the portable console\'s architecture and runs a demanding 3D '
  45       'simulation, all in JavaScript (11,097 lines).',
  46   'Mandreel':
  47       'Runs the 3D Bullet Physics Engine ported from C++ to JavaScript via '
  48       'Mandreel (277,377 lines).',
  49   'NavierStokes':
  50       '2D NavierStokes equations solver, heavily manipulates double precision '
  51       'arrays. Based on Oliver Hunt\'s code (387 lines).',
  52   'PdfJS':
  53       'Mozilla\'s PDF Reader implemented in JavaScript. It measures decoding '
  54       'and interpretation time (33,056 lines).',
  55   'RayTrace':
  56       'Ray tracer benchmark based on code by Adam Burmister (904 lines).',
  57   'RegExp':
  58       'Regular expression benchmark generated by extracting regular '
  59       'expression operations from 50 of the most popular web pages '
  60       '(1761 lines).',
  61   'Richards':
  62       'OS kernel simulation benchmark, originally written in BCPL by Martin '
  63       'Richards (539 lines).',
  64   'Splay':
  65       'Data manipulation benchmark that deals with splay trees and exercises '
  66       'the automatic memory management subsystem (394 lines).',
  67 }
  68
  69
  70 class _OctaneMeasurement(page_test.PageTest):
  71   def __init__(self):
  72     super(_OctaneMeasurement, self).__init__()
  73     self._power_metric = None
  74
  75   def CustomizeBrowserOptions(self, options):
  76     power.PowerMetric.CustomizeBrowserOptions(options)
  77
  78   def WillStartBrowser(self, platform):
  79     self._power_metric = power.PowerMetric(platform)
  80
  81   def WillNavigateToPage(self, page, tab):
  82     memory_stats = tab.browser.memory_stats
  83     if ('SystemTotalPhysicalMemory' in memory_stats and
  84         memory_stats['SystemTotalPhysicalMemory'] < 1 * _GB):
  85       skipBenchmarks = '"zlib"'
  86     else:
  87       skipBenchmarks = ''
  88     page.script_to_evaluate_on_commit = """
  89         var __results = [];
  90         var __real_log = window.console.log;
  91         window.console.log = function(msg) {
  92           __results.push(msg);
  93           __real_log.apply(this, [msg]);
  94         }
  95         skipBenchmarks = [%s]
  96         """ % (skipBenchmarks)
  97
  98   def DidNavigateToPage(self, page, tab):
  99     self._power_metric.Start(page, tab)
 100
 101   def ValidateAndMeasurePage(self, page, tab, results):
 102     tab.WaitForJavaScriptExpression('window.completed', 10)
 103     tab.WaitForJavaScriptExpression(
 104         '!document.getElementById("progress-bar-container")', 1200)
 105
 106     self._power_metric.Stop(page, tab)
 107     self._power_metric.AddResults(tab, results)
 108
 109     results_log = tab.EvaluateJavaScript('__results')
 110     all_scores = []
 111     for output in results_log:
 112       # Split the results into score and test name.
 113       # results log e.g., "Richards: 18343"
 114       score_and_name = output.split(': ', 2)
 115       assert len(score_and_name) == 2, \
 116         'Unexpected result format "%s"' % score_and_name
 117       if 'Skipped' not in score_and_name[1]:
 118         name = score_and_name[0]
 119         score = float(score_and_name[1])
 120         results.AddValue(scalar.ScalarValue(
 121             results.current_page, name, 'score', score, important=False,
 122             description=DESCRIPTIONS.get(name)))
 123
 124         # Collect all test scores to compute geometric mean.
 125         all_scores.append(score)
 126     total = statistics.GeometricMean(all_scores)
 127     results.AddSummaryValue(
 128         scalar.ScalarValue(None, 'Total.Score', 'score', total,
 129                            description='Geometric mean of the scores of each '
 130                                        'individual benchmark in the Octane '
 131                                        'benchmark collection.'))
 132
 133
 134 class Octane(perf_benchmark.PerfBenchmark):
 135   """Google's Octane JavaScript benchmark.
 136
 137   http://octane-benchmark.googlecode.com/svn/latest/index.html
 138   """
 139   test = _OctaneMeasurement
 140
 141   @classmethod
 142   def Name(cls):
 143     return 'octane'
 144
 145   def CreateStorySet(self, options):
 146     ps = story.StorySet(
 147       archive_data_file='../page_sets/data/octane.json',
 148       base_dir=os.path.dirname(os.path.abspath(__file__)),
 149       cloud_storage_bucket=story.PUBLIC_BUCKET)
 150     ps.AddStory(page_module.Page(
 151         'http://octane-benchmark.googlecode.com/svn/latest/index.html?auto=1',
 152         ps, ps.base_dir, make_javascript_deterministic=False))
 153     return ps