tools/perf/benchmarks/jetstream.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Runs Apple's JetStream benchmark.
   6
   7 JetStream combines a variety of JavaScript benchmarks, covering a variety of
   8 advanced workloads and programming techniques, and reports a single score that
   9 balances them using geometric mean.
  10
  11 Each benchmark measures a distinct workload, and no single optimization
  12 technique is sufficient to speed up all benchmarks. Latency tests measure that
  13 a web application can start up quickly, ramp up to peak performance quickly,
  14 and run smoothly without interruptions. Throughput tests measure the sustained
  15 peak performance of a web application, ignoring ramp-up time and spikes in
  16 smoothness. Some benchmarks demonstrate tradeoffs, and aggressive or
  17 specialized optimization for one benchmark might make another benchmark slower.
  18 """
  19
  20 import json
  21 import os
  22
  23 from core import perf_benchmark
  24
  25 from telemetry import benchmark
  26 from telemetry import page as page_module
  27 from telemetry.page import page_test
  28 from telemetry import story
  29 from telemetry.util import statistics
  30 from telemetry.value import list_of_scalar_values
  31
  32
  33 class _JetstreamMeasurement(page_test.PageTest):
  34   def __init__(self):
  35     super(_JetstreamMeasurement, self).__init__()
  36
  37   def WillNavigateToPage(self, page, tab):
  38     page.script_to_evaluate_on_commit = """
  39         var __results = [];
  40         var __real_log = window.console.log;
  41         window.console.log = function() {
  42           __results.push(Array.prototype.join.call(arguments, ' '));
  43           __real_log.apply(this, arguments);
  44         }
  45         """
  46
  47   def ValidateAndMeasurePage(self, page, tab, results):
  48     get_results_js = """
  49         (function() {
  50           for (var i = 0; i < __results.length; i++) {
  51             if (!__results[i].indexOf('Raw results: ')) return __results[i];
  52           }
  53           return null;
  54         })();
  55         """
  56
  57     tab.WaitForDocumentReadyStateToBeComplete()
  58     tab.EvaluateJavaScript('JetStream.start()')
  59     tab.WaitForJavaScriptExpression(get_results_js, 600)
  60
  61     result = tab.EvaluateJavaScript(get_results_js)
  62     result = json.loads(result.partition(': ')[2])
  63
  64     all_score_lists = []
  65     for k, v in result.iteritems():
  66       results.AddValue(list_of_scalar_values.ListOfScalarValues(
  67           results.current_page, k.replace('.', '_'), 'score', v['result'],
  68           important=False))
  69       # Collect all test scores to compute geometric mean.
  70       for i, score in enumerate(v['result']):
  71         if len(all_score_lists) <= i:
  72           all_score_lists.append([])
  73         all_score_lists[i].append(score)
  74     all_scores = []
  75     for score_list in all_score_lists:
  76       all_scores.append(statistics.GeometricMean(score_list))
  77     results.AddSummaryValue(list_of_scalar_values.ListOfScalarValues(
  78         None, 'Score', 'score', all_scores))
  79
  80
  81 @benchmark.Disabled('android', 'xp')  # crbug.com/381742
  82 class Jetstream(perf_benchmark.PerfBenchmark):
  83   test = _JetstreamMeasurement
  84
  85   @classmethod
  86   def Name(cls):
  87     return 'jetstream'
  88
  89   def CreateStorySet(self, options):
  90     ps = story.StorySet(
  91         archive_data_file='../page_sets/data/jetstream.json',
  92         base_dir=os.path.dirname(os.path.abspath(__file__)),
  93         cloud_storage_bucket=story.INTERNAL_BUCKET)
  94     ps.AddStory(page_module.Page(
  95         'http://browserbench.org/JetStream/', ps, ps.base_dir,
  96         make_javascript_deterministic=False))
  97     return ps