tools/perf/benchmarks/speedometer.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Apple's Speedometer performance benchmark.
   6
   7 Speedometer measures simulated user interactions in web applications.
   8
   9 The current benchmark uses TodoMVC to simulate user actions for adding,
  10 completing, and removing to-do items. Speedometer repeats the same actions using
  11 DOM APIs - a core set of web platform APIs used extensively in web applications-
  12 as well as six popular JavaScript frameworks: Ember.js, Backbone.js, jQuery,
  13 AngularJS, React, and Flight. Many of these frameworks are used on the most
  14 popular websites in the world, such as Facebook and Twitter. The performance of
  15 these types of operations depends on the speed of the DOM APIs, the JavaScript
  16 engine, CSS style resolution, layout, and other technologies.
  17 """
  18
  19 import os
  20
  21 from core import perf_benchmark
  22
  23 from telemetry import page as page_module
  24 from telemetry.page import page_test
  25 from telemetry import story
  26 from telemetry.value import list_of_scalar_values
  27
  28 from metrics import keychain_metric
  29
  30
  31 class SpeedometerMeasurement(page_test.PageTest):
  32   enabled_suites = [
  33     'VanillaJS-TodoMVC',
  34     'EmberJS-TodoMVC',
  35     'BackboneJS-TodoMVC',
  36     'jQuery-TodoMVC',
  37     'AngularJS-TodoMVC',
  38     'React-TodoMVC',
  39     'FlightJS-TodoMVC'
  40   ]
  41
  42   def __init__(self):
  43     super(SpeedometerMeasurement, self).__init__()
  44
  45   def CustomizeBrowserOptions(self, options):
  46     keychain_metric.KeychainMetric.CustomizeBrowserOptions(options)
  47
  48   def ValidateAndMeasurePage(self, page, tab, results):
  49     tab.WaitForDocumentReadyStateToBeComplete()
  50     iterationCount = 10
  51     # A single iteration on android takes ~75 seconds, the benchmark times out
  52     # when running for 10 iterations.
  53     if tab.browser.platform.GetOSName() == 'android':
  54       iterationCount = 3
  55
  56     tab.ExecuteJavaScript("""
  57         // Store all the results in the benchmarkClient
  58         benchmarkClient._measuredValues = []
  59         benchmarkClient.didRunSuites = function(measuredValues) {
  60           benchmarkClient._measuredValues.push(measuredValues);
  61           benchmarkClient._timeValues.push(measuredValues.total);
  62         };
  63         benchmarkClient.iterationCount = %d;
  64         startTest();
  65         """ % iterationCount)
  66     tab.WaitForJavaScriptExpression(
  67         'benchmarkClient._finishedTestCount == benchmarkClient.testsCount', 600)
  68     results.AddValue(list_of_scalar_values.ListOfScalarValues(
  69         page, 'Total', 'ms',
  70         tab.EvaluateJavaScript('benchmarkClient._timeValues'), important=True))
  71
  72     # Extract the timings for each suite
  73     for suite_name in self.enabled_suites:
  74       results.AddValue(list_of_scalar_values.ListOfScalarValues(
  75           page, suite_name, 'ms',
  76           tab.EvaluateJavaScript("""
  77               var suite_times = [];
  78               for(var i = 0; i < benchmarkClient.iterationCount; i++) {
  79                 suite_times.push(
  80                     benchmarkClient._measuredValues[i].tests['%s'].total);
  81               };
  82               suite_times;
  83               """ % suite_name), important=False))
  84     keychain_metric.KeychainMetric().AddResults(tab, results)
  85
  86 class Speedometer(perf_benchmark.PerfBenchmark):
  87   test = SpeedometerMeasurement
  88
  89   @classmethod
  90   def Name(cls):
  91     return 'speedometer'
  92
  93   def CreateStorySet(self, options):
  94     ps = story.StorySet(
  95         base_dir=os.path.dirname(os.path.abspath(__file__)),
  96         archive_data_file='../page_sets/data/speedometer.json',
  97         cloud_storage_bucket=story.PUBLIC_BUCKET)
  98     ps.AddStory(page_module.Page(
  99         'http://browserbench.org/Speedometer/', ps, ps.base_dir,
 100         make_javascript_deterministic=False))
 101     return ps