tools/perf/benchmarks/page_cycler.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 from core import perf_benchmark
   6
   7 from measurements import page_cycler
   8 import page_sets
   9 from telemetry import benchmark
  10
  11
  12 class _PageCycler(perf_benchmark.PerfBenchmark):
  13   options = {'pageset_repeat': 6}
  14   cold_load_percent = 50  # % of page visits for which a cold load is forced
  15
  16   @classmethod
  17   def Name(cls):
  18     return 'page_cycler'
  19
  20   @classmethod
  21   def AddBenchmarkCommandLineArgs(cls, parser):
  22     parser.add_option('--report-speed-index',
  23         action='store_true',
  24         help='Enable the speed index metric.')
  25
  26   @classmethod
  27   def ValueCanBeAddedPredicate(cls, _, is_first_result):
  28     return cls.cold_load_percent > 0 or not is_first_result
  29
  30   def CreatePageTest(self, options):
  31     return page_cycler.PageCycler(
  32         page_repeat = options.page_repeat,
  33         pageset_repeat = options.pageset_repeat,
  34         cold_load_percent = self.cold_load_percent,
  35         report_speed_index = options.report_speed_index)
  36
  37
  38 # This is an old page set, we intend to remove it after more modern benchmarks
  39 # work on CrOS.
  40 @benchmark.Enabled('chromeos')
  41 class PageCyclerDhtml(_PageCycler):
  42   """Benchmarks for various DHTML operations like simple animations."""
  43   page_set = page_sets.DhtmlPageSet
  44
  45   @classmethod
  46   def Name(cls):
  47     return 'page_cycler.dhtml'
  48
  49
  50 class PageCyclerIntlArFaHe(_PageCycler):
  51   """Page load time for a variety of pages in Arabic, Farsi and Hebrew.
  52
  53   Runs against pages recorded in April, 2013.
  54   """
  55   page_set = page_sets.IntlArFaHePageSet
  56
  57   @classmethod
  58   def Name(cls):
  59     return 'page_cycler.intl_ar_fa_he'
  60
  61
  62 @benchmark.Disabled('win')  # crbug.com/366715
  63 class PageCyclerIntlEsFrPtBr(_PageCycler):
  64   """Page load time for a pages in Spanish, French and Brazilian Portuguese.
  65
  66   Runs against pages recorded in April, 2013.
  67   """
  68   page_set = page_sets.IntlEsFrPtBrPageSet
  69
  70   @classmethod
  71   def Name(cls):
  72     return 'page_cycler.intl_es_fr_pt-BR'
  73
  74
  75 class PageCyclerIntlHiRu(_PageCycler):
  76   """Page load time benchmark for a variety of pages in Hindi and Russian.
  77
  78   Runs against pages recorded in April, 2013.
  79   """
  80   page_set = page_sets.IntlHiRuPageSet
  81
  82   @classmethod
  83   def Name(cls):
  84     return 'page_cycler.intl_hi_ru'
  85
  86
  87 @benchmark.Disabled('android', 'win')  # crbug.com/379564, crbug.com/434366
  88 class PageCyclerIntlJaZh(_PageCycler):
  89   """Page load time benchmark for a variety of pages in Japanese and Chinese.
  90
  91   Runs against pages recorded in April, 2013.
  92   """
  93   page_set = page_sets.IntlJaZhPageSet
  94
  95   @classmethod
  96   def Name(cls):
  97     return 'page_cycler.intl_ja_zh'
  98
  99
 100 @benchmark.Disabled('xp', 'android')  # crbug.com/434366, crbug.com/506903
 101 class PageCyclerIntlKoThVi(_PageCycler):
 102   """Page load time for a variety of pages in Korean, Thai and Vietnamese.
 103
 104   Runs against pages recorded in April, 2013.
 105   """
 106   page_set = page_sets.IntlKoThViPageSet
 107
 108   @classmethod
 109   def Name(cls):
 110     return 'page_cycler.intl_ko_th_vi'
 111
 112
 113 class PageCyclerMorejs(_PageCycler):
 114   """Page load for a variety of pages that were JavaScript heavy in 2009."""
 115   page_set = page_sets.MorejsPageSet
 116
 117   @classmethod
 118   def Name(cls):
 119     return 'page_cycler.morejs'
 120
 121
 122 # This is an old page set, we intend to remove it after more modern benchmarks
 123 # work on CrOS.
 124 @benchmark.Enabled('chromeos')
 125 class PageCyclerMoz(_PageCycler):
 126   """Page load for mozilla's original page set. Recorded in December 2000."""
 127   page_set = page_sets.MozPageSet
 128
 129   @classmethod
 130   def Name(cls):
 131     return 'page_cycler.moz'
 132
 133
 134 # Win, mac, linux: crbug.com/353260
 135 # Android: crbug.com/473161
 136 @benchmark.Disabled('linux', 'win', 'mac', 'android')
 137 class PageCyclerNetsimTop10(_PageCycler):
 138   """Measures load time of the top 10 sites under simulated cable network.
 139
 140   Recorded in June, 2013.  Pages are loaded under the simplisticly simulated
 141   bandwidth and RTT constraints of a cable modem (5Mbit/s down, 1Mbit/s up,
 142   28ms RTT). Contention is realistically simulated, but slow start is not.
 143   DNS lookups are 'free'.
 144   """
 145   tag = 'netsim'
 146   page_set = page_sets.Top10PageSet
 147   options = {
 148       'extra_wpr_args_as_string': '--shaping_type=proxy --net=cable',
 149       'pageset_repeat': 6,
 150   }
 151   cold_load_percent = 100
 152
 153   @classmethod
 154   def Name(cls):
 155     return 'page_cycler.netsim.top_10'
 156
 157   def CreatePageTest(self, options):
 158     return page_cycler.PageCycler(
 159         page_repeat = options.page_repeat,
 160         pageset_repeat = options.pageset_repeat,
 161         cold_load_percent = self.cold_load_percent,
 162         report_speed_index = options.report_speed_index,
 163         clear_cache_before_each_run = True)
 164
 165 @benchmark.Enabled('android')
 166 class PageCyclerTop10Mobile(_PageCycler):
 167   """Page load time benchmark for the top 10 mobile web pages.
 168
 169   Runs against pages recorded in November, 2013.
 170   """
 171
 172   @classmethod
 173   def Name(cls):
 174     return 'page_cycler.top_10_mobile'
 175
 176   def CreateStorySet(self, options):
 177     # Disable the taobao.com page since it's crashing. crbug.com/509690
 178     stories = page_sets.Top10MobilePageSet(run_no_page_interactions=True)
 179     found = next((x for x in stories if 'taobao.com' in x.url), None)
 180     if found:
 181       stories.RemoveStory(found)
 182     return stories
 183
 184 @benchmark.Disabled
 185 class PageCyclerKeyMobileSites(_PageCycler):
 186   """Page load time benchmark for key mobile sites."""
 187   page_set = page_sets.KeyMobileSitesPageSet
 188
 189   @classmethod
 190   def Name(cls):
 191     return 'page_cycler.key_mobile_sites_smooth'
 192
 193
 194 @benchmark.Disabled('android')  # crbug.com/357326
 195 class PageCyclerToughLayoutCases(_PageCycler):
 196   """Page loading for the slowest layouts observed in the Alexa top 1 million.
 197
 198   Recorded in July 2013.
 199   """
 200   page_set = page_sets.ToughLayoutCasesPageSet
 201
 202   @classmethod
 203   def Name(cls):
 204     return 'page_cycler.tough_layout_cases'
 205
 206
 207 # crbug.com/273986: This test is flakey on Windows.
 208 @benchmark.Disabled('win')
 209 class PageCyclerTypical25(_PageCycler):
 210   """Page load time benchmark for a 25 typical web pages.
 211
 212   Designed to represent typical, not highly optimized or highly popular web
 213   sites. Runs against pages recorded in June, 2014.
 214   """
 215
 216   @classmethod
 217   def Name(cls):
 218     return 'page_cycler.typical_25'
 219
 220   def CreateStorySet(self, options):
 221     return page_sets.Typical25PageSet(run_no_page_interactions=True)
 222
 223 @benchmark.Disabled('reference', 'android')
 224 class PageCyclerBasicOopifIsolated(_PageCycler):
 225   """ A benchmark measuring performance of out-of-process iframes. """
 226   page_set = page_sets.OopifBasicPageSet
 227
 228   @classmethod
 229   def Name(cls):
 230     return 'page_cycler_site_isolation.basic_oopif'
 231
 232   def SetExtraBrowserOptions(self, options):
 233     options.AppendExtraBrowserArgs(['--site-per-process'])
 234
 235 class PageCyclerBasicOopif(_PageCycler):
 236   """ A benchmark measuring performance of the out-of-process iframes page
 237   set, without running in out-of-process iframes mode.. """
 238   page_set = page_sets.OopifBasicPageSet
 239
 240   @classmethod
 241   def Name(cls):
 242     return 'page_cycler.basic_oopif'
 243
 244 @benchmark.Disabled # crbug.com/443730
 245 class PageCyclerBigJs(_PageCycler):
 246   page_set = page_sets.BigJsPageSet
 247   @classmethod
 248   def Name(cls):
 249     return 'page_cycler.big_js'