tools/perf/benchmarks/page_cycler.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 from core import perf_benchmark
   6
   7 from measurements import page_cycler
   8 import page_sets
   9 from telemetry import benchmark
  10
  11
  12 class _PageCycler(perf_benchmark.PerfBenchmark):
  13   options = {'pageset_repeat': 6}
  14   cold_load_percent = 50  # % of page visits for which a cold load is forced
  15
  16   @classmethod
  17   def Name(cls):
  18     return 'page_cycler'
  19
  20   @classmethod
  21   def AddBenchmarkCommandLineArgs(cls, parser):
  22     parser.add_option('--report-speed-index',
  23         action='store_true',
  24         help='Enable the speed index metric.')
  25
  26   @classmethod
  27   def ValueCanBeAddedPredicate(cls, _, is_first_result):
  28     return cls.cold_load_percent > 0 or not is_first_result
  29
  30   def CreatePageTest(self, options):
  31     return page_cycler.PageCycler(
  32         page_repeat = options.page_repeat,
  33         pageset_repeat = options.pageset_repeat,
  34         cold_load_percent = self.cold_load_percent,
  35         report_speed_index = options.report_speed_index)
  36
  37
  38 # This is an old page set, we intend to remove it after more modern benchmarks
  39 # work on CrOS.
  40 @benchmark.Enabled('chromeos')
  41 class PageCyclerDhtml(_PageCycler):
  42   """Benchmarks for various DHTML operations like simple animations."""
  43   page_set = page_sets.DhtmlPageSet
  44
  45   @classmethod
  46   def Name(cls):
  47     return 'page_cycler.dhtml'
  48
  49
  50 class PageCyclerIntlArFaHe(_PageCycler):
  51   """Page load time for a variety of pages in Arabic, Farsi and Hebrew.
  52
  53   Runs against pages recorded in April, 2013.
  54   """
  55   page_set = page_sets.IntlArFaHePageSet
  56
  57   @classmethod
  58   def Name(cls):
  59     return 'page_cycler.intl_ar_fa_he'
  60
  61
  62 @benchmark.Disabled('win')  # crbug.com/366715
  63 class PageCyclerIntlEsFrPtBr(_PageCycler):
  64   """Page load time for a pages in Spanish, French and Brazilian Portuguese.
  65
  66   Runs against pages recorded in April, 2013.
  67   """
  68   page_set = page_sets.IntlEsFrPtBrPageSet
  69
  70   @classmethod
  71   def Name(cls):
  72     return 'page_cycler.intl_es_fr_pt-BR'
  73
  74
  75 class PageCyclerIntlHiRu(_PageCycler):
  76   """Page load time benchmark for a variety of pages in Hindi and Russian.
  77
  78   Runs against pages recorded in April, 2013.
  79   """
  80   page_set = page_sets.IntlHiRuPageSet
  81
  82   @classmethod
  83   def Name(cls):
  84     return 'page_cycler.intl_hi_ru'
  85
  86
  87 @benchmark.Disabled('android', 'win')  # crbug.com/379564, crbug.com/434366
  88 class PageCyclerIntlJaZh(_PageCycler):
  89   """Page load time benchmark for a variety of pages in Japanese and Chinese.
  90
  91   Runs against pages recorded in April, 2013.
  92   """
  93   page_set = page_sets.IntlJaZhPageSet
  94
  95   @classmethod
  96   def Name(cls):
  97     return 'page_cycler.intl_ja_zh'
  98
  99
 100 @benchmark.Disabled('xp', 'android')  # crbug.com/434366, crbug.com/506903
 101 class PageCyclerIntlKoThVi(_PageCycler):
 102   """Page load time for a variety of pages in Korean, Thai and Vietnamese.
 103
 104   Runs against pages recorded in April, 2013.
 105   """
 106   page_set = page_sets.IntlKoThViPageSet
 107
 108   @classmethod
 109   def Name(cls):
 110     return 'page_cycler.intl_ko_th_vi'
 111
 112
 113 class PageCyclerMorejs(_PageCycler):
 114   """Page load for a variety of pages that were JavaScript heavy in 2009."""
 115   page_set = page_sets.MorejsPageSet
 116
 117   @classmethod
 118   def Name(cls):
 119     return 'page_cycler.morejs'
 120
 121
 122 # This is an old page set, we intend to remove it after more modern benchmarks
 123 # work on CrOS.
 124 @benchmark.Enabled('chromeos')
 125 class PageCyclerMoz(_PageCycler):
 126   """Page load for mozilla's original page set. Recorded in December 2000."""
 127   page_set = page_sets.MozPageSet
 128
 129   @classmethod
 130   def Name(cls):
 131     return 'page_cycler.moz'
 132
 133
 134 # Win, mac, linux: crbug.com/353260
 135 # Android: crbug.com/473161
 136 @benchmark.Disabled('linux', 'win', 'mac', 'android')
 137 class PageCyclerNetsimTop10(_PageCycler):
 138   """Measures load time of the top 10 sites under simulated cable network.
 139
 140   Recorded in June, 2013.  Pages are loaded under the simplisticly simulated
 141   bandwidth and RTT constraints of a cable modem (5Mbit/s down, 1Mbit/s up,
 142   28ms RTT). Contention is realistically simulated, but slow start is not.
 143   DNS lookups are 'free'.
 144   """
 145   tag = 'netsim'
 146   page_set = page_sets.Top10PageSet
 147   options = {
 148       'extra_wpr_args_as_string': '--shaping_type=proxy --net=cable',
 149       'pageset_repeat': 6,
 150   }
 151   cold_load_percent = 100
 152
 153   @classmethod
 154   def Name(cls):
 155     return 'page_cycler.netsim.top_10'
 156
 157   def CreatePageTest(self, options):
 158     return page_cycler.PageCycler(
 159         page_repeat = options.page_repeat,
 160         pageset_repeat = options.pageset_repeat,
 161         cold_load_percent = self.cold_load_percent,
 162         report_speed_index = options.report_speed_index,
 163         clear_cache_before_each_run = True)
 164
 165 @benchmark.Enabled('android')
 166 class PageCyclerTop10Mobile(_PageCycler):
 167   """Page load time benchmark for the top 10 mobile web pages.
 168
 169   Runs against pages recorded in November, 2013.
 170   """
 171
 172   @classmethod
 173   def Name(cls):
 174     return 'page_cycler.top_10_mobile'
 175
 176   def CreateStorySet(self, options):
 177     # Disable the taobao.com page since it's crashing. crbug.com/509690
 178     stories = page_sets.Top10MobilePageSet(run_no_page_interactions=True)
 179     found = next((x for x in stories if 'taobao.com' in x.url), None)
 180     if found:
 181       stories.RemoveStory(found)
 182     return stories
 183
 184 @benchmark.Disabled
 185 class PageCyclerKeyMobileSites(_PageCycler):
 186   """Page load time benchmark for key mobile sites."""
 187   page_set = page_sets.KeyMobileSitesPageSet
 188
 189   @classmethod
 190   def Name(cls):
 191     return 'page_cycler.key_mobile_sites_smooth'
 192
 193
 194 @benchmark.Disabled('android')  # crbug.com/357326
 195 class PageCyclerToughLayoutCases(_PageCycler):
 196   """Page loading for the slowest layouts observed in the Alexa top 1 million.
 197
 198   Recorded in July 2013.
 199   """
 200   page_set = page_sets.ToughLayoutCasesPageSet
 201
 202   @classmethod
 203   def Name(cls):
 204     return 'page_cycler.tough_layout_cases'
 205
 206
 207 # crbug.com/273986: This test is flakey on Windows Chrome.
 208 @benchmark.Enabled('android', 'chromeos', 'linux', 'ios', 'mac',
 209                    'mandoline-release', 'mandoline-debug')
 210 class PageCyclerTypical25(_PageCycler):
 211   """Page load time benchmark for a 25 typical web pages.
 212
 213   Designed to represent typical, not highly optimized or highly popular web
 214   sites. Runs against pages recorded in June, 2014.
 215   """
 216
 217   @classmethod
 218   def Name(cls):
 219     return 'page_cycler.typical_25'
 220
 221   def CreateStorySet(self, options):
 222     return page_sets.Typical25PageSet(run_no_page_interactions=True)
 223
 224
 225 @benchmark.Disabled('reference', 'android')
 226 class PageCyclerBasicOopifIsolated(_PageCycler):
 227   """ A benchmark measuring performance of out-of-process iframes. """
 228   page_set = page_sets.OopifBasicPageSet
 229
 230   @classmethod
 231   def Name(cls):
 232     return 'page_cycler_site_isolation.basic_oopif'
 233
 234   def SetExtraBrowserOptions(self, options):
 235     options.AppendExtraBrowserArgs(['--site-per-process'])
 236
 237
 238 @benchmark.Disabled('reference')  # crbug.com/523346
 239 class PageCyclerBasicOopif(_PageCycler):
 240   """ A benchmark measuring performance of the out-of-process iframes page
 241   set, without running in out-of-process iframes mode.. """
 242   page_set = page_sets.OopifBasicPageSet
 243
 244   @classmethod
 245   def Name(cls):
 246     return 'page_cycler.basic_oopif'
 247
 248
 249 @benchmark.Disabled  # crbug.com/443730
 250 class PageCyclerBigJs(_PageCycler):
 251   page_set = page_sets.BigJsPageSet
 252   @classmethod
 253   def Name(cls):
 254     return 'page_cycler.big_js'