tools/perf/benchmarks/service_worker.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import collections
   6 import page_sets
   7 import re
   8
   9 from core import perf_benchmark
  10
  11 from telemetry import benchmark
  12 from telemetry.core import util
  13 from telemetry.page import action_runner
  14 from telemetry.page import page_test
  15 from telemetry.timeline import async_slice as async_slice_module
  16 from telemetry.timeline import slice as slice_module
  17 from telemetry.value import scalar
  18
  19 from measurements import timeline_controller
  20 from metrics import speedindex
  21
  22
  23 class _ServiceWorkerTimelineMetric(object):
  24   def AddResultsOfCounters(self, process, counter_regex_string, results):
  25     counter_filter = re.compile(counter_regex_string)
  26     for counter_name, counter in process.counters.iteritems():
  27       if not counter_filter.search(counter_name):
  28         continue
  29
  30       total = sum(counter.totals)
  31
  32       # Results objects cannot contain the '.' character, so remove that here.
  33       sanitized_counter_name = counter_name.replace('.', '_')
  34
  35       results.AddValue(scalar.ScalarValue(
  36           results.current_page, sanitized_counter_name, 'count', total))
  37       results.AddValue(scalar.ScalarValue(
  38           results.current_page, sanitized_counter_name + '_avg', 'count',
  39           total / float(len(counter.totals))))
  40
  41   def AddResultsOfEvents(
  42       self, process, thread_regex_string, event_regex_string, results):
  43     thread_filter = re.compile(thread_regex_string)
  44     event_filter = re.compile(event_regex_string)
  45
  46     for thread in process.threads.itervalues():
  47       thread_name = thread.name.replace('/', '_')
  48       if not thread_filter.search(thread_name):
  49         continue
  50
  51       filtered_events = []
  52       for event in thread.IterAllEvents():
  53         event_name = event.name.replace('.', '_')
  54         if event_filter.search(event_name):
  55           filtered_events.append(event)
  56
  57       async_events_by_name = collections.defaultdict(list)
  58       sync_events_by_name = collections.defaultdict(list)
  59       for event in filtered_events:
  60         if isinstance(event, async_slice_module.AsyncSlice):
  61           async_events_by_name[event.name].append(event)
  62         elif isinstance(event, slice_module.Slice):
  63           sync_events_by_name[event.name].append(event)
  64
  65       for event_name, event_group in async_events_by_name.iteritems():
  66         times = [e.duration for e in event_group]
  67         self._AddResultOfEvent(thread_name, event_name, times, results)
  68
  69       for event_name, event_group in sync_events_by_name.iteritems():
  70         times = [e.self_time for e in event_group]
  71         self._AddResultOfEvent(thread_name, event_name, times, results)
  72
  73   def _AddResultOfEvent(self, thread_name, event_name, times, results):
  74     total = sum(times)
  75     biggest_jank = max(times)
  76
  77     # Results objects cannot contain the '.' character, so remove that here.
  78     sanitized_event_name = event_name.replace('.', '_')
  79
  80     full_name = thread_name + '|' + sanitized_event_name
  81     results.AddValue(scalar.ScalarValue(
  82         results.current_page, full_name, 'ms', total))
  83     results.AddValue(scalar.ScalarValue(
  84         results.current_page, full_name + '_max', 'ms', biggest_jank))
  85     results.AddValue(scalar.ScalarValue(
  86         results.current_page, full_name + '_avg', 'ms', total / len(times)))
  87
  88
  89 class _ServiceWorkerMeasurement(page_test.PageTest):
  90   """Measure Speed Index and TRACE_EVENTs"""
  91
  92   def __init__(self):
  93     super(_ServiceWorkerMeasurement, self).__init__()
  94     self._timeline_controller = timeline_controller.TimelineController()
  95     self._speed_index = speedindex.SpeedIndexMetric()
  96     self._page_open_times = collections.defaultdict(int)
  97
  98   def CustomizeBrowserOptions(self, options):
  99     options.AppendExtraBrowserArgs([
 100         '--enable-experimental-web-platform-features'
 101       ])
 102
 103   def WillNavigateToPage(self, page, tab):
 104     self._timeline_controller.SetUp(page, tab)
 105     self._timeline_controller.Start(tab)
 106     self._speed_index.Start(page, tab)
 107
 108   def ValidateAndMeasurePage(self, page, tab, results):
 109     runner = action_runner.ActionRunner(tab)
 110     # timeline_controller requires creation of at least a single interaction
 111     # record. service_worker should be refactored to follow the
 112     # timeline_based_measurement or it should not re-use timeline_controller
 113     # logic for start & stop tracing.
 114     with runner.CreateInteraction('_DummyInteraction'):
 115       pass
 116     tab.WaitForDocumentReadyStateToBeComplete(40)
 117     self._timeline_controller.Stop(tab, results)
 118
 119     # Retrieve TRACE_EVENTs
 120     timeline_metric = _ServiceWorkerTimelineMetric()
 121     browser_process = self._timeline_controller.model.browser_process
 122     filter_text = '(RegisterServiceWorker|'\
 123                   'UnregisterServiceWorker|'\
 124                   'ProcessAllocate|'\
 125                   'FindRegistrationForDocument|'\
 126                   'DispatchFetchEvent)'
 127     timeline_metric.AddResultsOfEvents(
 128         browser_process, 'IOThread', filter_text , results)
 129
 130     # Record Speed Index
 131     def SpeedIndexIsFinished():
 132       return self._speed_index.IsFinished(tab)
 133     util.WaitFor(SpeedIndexIsFinished, 60)
 134     self._speed_index.Stop(page, tab)
 135     # Distinguish the first and second load from the subsequent loads
 136     url = str(page)
 137     chart_prefix = 'page_load'
 138     self._page_open_times[url] += 1
 139     if self._page_open_times[url] == 1:
 140       chart_prefix += '_1st'
 141     elif self._page_open_times[url] == 2:
 142       chart_prefix += '_2nd'
 143     else:
 144       chart_prefix += '_later'
 145     self._speed_index.AddResults(tab, results, chart_prefix)
 146
 147
 148 class _ServiceWorkerMicroBenchmarkMeasurement(page_test.PageTest):
 149   """Measure JS land values and TRACE_EVENTs"""
 150
 151   def __init__(self):
 152     super(_ServiceWorkerMicroBenchmarkMeasurement, self).__init__()
 153     self._timeline_controller = timeline_controller.TimelineController()
 154
 155   def CustomizeBrowserOptions(self, options):
 156     options.AppendExtraBrowserArgs([
 157         '--enable-experimental-web-platform-features'
 158       ])
 159
 160   def WillNavigateToPage(self, page, tab):
 161     self._timeline_controller.SetUp(page, tab)
 162     self._timeline_controller.Start(tab)
 163
 164   def ValidateAndMeasurePage(self, page, tab, results):
 165     runner = action_runner.ActionRunner(tab)
 166     # timeline_controller requires creation of at least a single interaction
 167     # record. service_worker should be refactored to follow the
 168     # timeline_based_measurement or it should not re-use timeline_controller
 169     # logic for start & stop tracing.
 170     with runner.CreateInteraction('_DummyInteraction'):
 171       pass
 172     tab.WaitForJavaScriptExpression('window.done', 40)
 173     self._timeline_controller.Stop(tab, results)
 174
 175     # Measure JavaScript-land
 176     json = tab.EvaluateJavaScript('window.results || {}')
 177     for key, value in json.iteritems():
 178       results.AddValue(scalar.ScalarValue(
 179           results.current_page, key, value['units'], value['value']))
 180
 181     # Retrieve TRACE_EVENTs
 182     timeline_metric = _ServiceWorkerTimelineMetric()
 183     browser_process = self._timeline_controller.model.browser_process
 184     filter_text = '(RegisterServiceWorker|'\
 185                   'UnregisterServiceWorker|'\
 186                   'ProcessAllocate|'\
 187                   'FindRegistrationForDocument|'\
 188                   'DispatchFetchEvent)'
 189     timeline_metric.AddResultsOfEvents(
 190         browser_process, 'IOThread', filter_text , results)
 191
 192
 193 class ServiceWorkerPerfTest(perf_benchmark.PerfBenchmark):
 194   """Performance test on public applications using ServiceWorker"""
 195   test = _ServiceWorkerMeasurement
 196   page_set = page_sets.ServiceWorkerPageSet
 197
 198   @classmethod
 199   def Name(cls):
 200     return 'service_worker.service_worker'
 201
 202
 203 # Disabled due to redness on the tree. crbug.com/442752
 204 # TODO(horo): Enable after the reference build newer than M39 will be rolled.
 205 @benchmark.Disabled('reference')
 206 class ServiceWorkerMicroBenchmarkPerfTest(perf_benchmark.PerfBenchmark):
 207   """This test measures the performance of pages using ServiceWorker.
 208
 209   As a page set, two benchamrk pages (many registration, many concurrent
 210   fetching) and one application (Trained-to-thrill:
 211   https://jakearchibald.github.io/trained-to-thrill/) are included. Execution
 212   time of these pages will be shown as Speed Index, and TRACE_EVENTs are
 213   subsidiary information to know more detail performance regression.
 214   """
 215   test = _ServiceWorkerMicroBenchmarkMeasurement
 216   page_set = page_sets.ServiceWorkerMicroBenchmarkPageSet
 217   @classmethod
 218   def Name(cls):
 219     return 'service_worker.service_worker_micro_benchmark'
 220