tools/perf/benchmarks/page_cycler.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 from core import perf_benchmark
   6
   7 from measurements import page_cycler
   8 from telemetry import benchmark
   9 import page_sets
  10
  11
  12 class _PageCycler(perf_benchmark.PerfBenchmark):
  13   options = {'pageset_repeat': 6}
  14   cold_load_percent = 50  # % of page visits for which a cold load is forced
  15
  16   @classmethod
  17   def Name(cls):
  18     return 'page_cycler'
  19
  20   @classmethod
  21   def AddBenchmarkCommandLineArgs(cls, parser):
  22     parser.add_option('--report-speed-index',
  23         action='store_true',
  24         help='Enable the speed index metric.')
  25
  26   @classmethod
  27   def ValueCanBeAddedPredicate(cls, _, is_first_result):
  28     return cls.cold_load_percent > 0 or not is_first_result
  29
  30   def CreatePageTest(self, options):
  31     return page_cycler.PageCycler(
  32         page_repeat = options.page_repeat,
  33         pageset_repeat = options.pageset_repeat,
  34         cold_load_percent = self.cold_load_percent,
  35         report_speed_index = options.report_speed_index)
  36
  37
  38 # This is an old page set, we intend to remove it after more modern benchmarks
  39 # work on CrOS.
  40 @benchmark.Enabled('chromeos')
  41 class PageCyclerDhtml(_PageCycler):
  42   """Benchmarks for various DHTML operations like simple animations."""
  43   page_set = page_sets.DhtmlPageSet
  44
  45   @classmethod
  46   def Name(cls):
  47     return 'page_cycler.dhtml'
  48
  49
  50 class PageCyclerIntlArFaHe(_PageCycler):
  51   """Page load time for a variety of pages in Arabic, Farsi and Hebrew.
  52
  53   Runs against pages recorded in April, 2013.
  54   """
  55   page_set = page_sets.IntlArFaHePageSet
  56
  57   @classmethod
  58   def Name(cls):
  59     return 'page_cycler.intl_ar_fa_he'
  60
  61
  62 @benchmark.Disabled('win')  # crbug.com/366715
  63 class PageCyclerIntlEsFrPtBr(_PageCycler):
  64   """Page load time for a pages in Spanish, French and Brazilian Portuguese.
  65
  66   Runs against pages recorded in April, 2013.
  67   """
  68   page_set = page_sets.IntlEsFrPtBrPageSet
  69
  70   @classmethod
  71   def Name(cls):
  72     return 'page_cycler.intl_es_fr_pt-BR'
  73
  74
  75 class PageCyclerIntlHiRu(_PageCycler):
  76   """Page load time benchmark for a variety of pages in Hindi and Russian.
  77
  78   Runs against pages recorded in April, 2013.
  79   """
  80   page_set = page_sets.IntlHiRuPageSet
  81
  82   @classmethod
  83   def Name(cls):
  84     return 'page_cycler.intl_hi_ru'
  85
  86
  87 @benchmark.Disabled('android', 'win')  # crbug.com/379564, crbug.com/434366
  88 class PageCyclerIntlJaZh(_PageCycler):
  89   """Page load time benchmark for a variety of pages in Japanese and Chinese.
  90
  91   Runs against pages recorded in April, 2013.
  92   """
  93   page_set = page_sets.IntlJaZhPageSet
  94
  95   @classmethod
  96   def Name(cls):
  97     return 'page_cycler.intl_ja_zh'
  98
  99
 100 @benchmark.Disabled('xp')  # crbug.com/434366
 101 class PageCyclerIntlKoThVi(_PageCycler):
 102   """Page load time for a variety of pages in Korean, Thai and Vietnamese.
 103
 104   Runs against pages recorded in April, 2013.
 105   """
 106   page_set = page_sets.IntlKoThViPageSet
 107
 108   @classmethod
 109   def Name(cls):
 110     return 'page_cycler.intl_ko_th_vi'
 111
 112
 113 class PageCyclerMorejs(_PageCycler):
 114   """Page load for a variety of pages that were JavaScript heavy in 2009."""
 115   page_set = page_sets.MorejsPageSet
 116
 117   @classmethod
 118   def Name(cls):
 119     return 'page_cycler.morejs'
 120
 121
 122 # This is an old page set, we intend to remove it after more modern benchmarks
 123 # work on CrOS.
 124 @benchmark.Enabled('chromeos')
 125 class PageCyclerMoz(_PageCycler):
 126   """Page load for mozilla's original page set. Recorded in December 2000."""
 127   page_set = page_sets.MozPageSet
 128
 129   @classmethod
 130   def Name(cls):
 131     return 'page_cycler.moz'
 132
 133
 134 # Win, mac, linux: crbug.com/353260
 135 # Android: crbug.com/473161
 136 @benchmark.Disabled('linux', 'win', 'mac', 'android')
 137 class PageCyclerNetsimTop10(_PageCycler):
 138   """Measures load time of the top 10 sites under simulated cable network.
 139
 140   Recorded in June, 2013.  Pages are loaded under the simplisticly simulated
 141   bandwidth and RTT constraints of a cable modem (5Mbit/s down, 1Mbit/s up,
 142   28ms RTT). Contention is realistically simulated, but slow start is not.
 143   DNS lookups are 'free'.
 144   """
 145   tag = 'netsim'
 146   page_set = page_sets.Top10PageSet
 147   options = {
 148       'extra_wpr_args_as_string': '--shaping_type=proxy --net=cable',
 149       'pageset_repeat': 6,
 150   }
 151   cold_load_percent = 100
 152
 153   @classmethod
 154   def Name(cls):
 155     return 'page_cycler.netsim.top_10'
 156
 157   def CreatePageTest(self, options):
 158     return page_cycler.PageCycler(
 159         page_repeat = options.page_repeat,
 160         pageset_repeat = options.pageset_repeat,
 161         cold_load_percent = self.cold_load_percent,
 162         report_speed_index = options.report_speed_index,
 163         clear_cache_before_each_run = True)
 164
 165
 166 @benchmark.Enabled('android')
 167 class PageCyclerTop10Mobile(_PageCycler):
 168   """Page load time benchmark for the top 10 mobile web pages.
 169
 170   Runs against pages recorded in November, 2013.
 171   """
 172
 173   @classmethod
 174   def Name(cls):
 175     return 'page_cycler.top_10_mobile'
 176
 177   def CreatePageSet(self, options):
 178     return page_sets.Top10MobilePageSet(run_no_page_interactions=True)
 179
 180
 181 @benchmark.Disabled
 182 class PageCyclerKeyMobileSites(_PageCycler):
 183   """Page load time benchmark for key mobile sites."""
 184   page_set = page_sets.KeyMobileSitesPageSet
 185
 186   @classmethod
 187   def Name(cls):
 188     return 'page_cycler.key_mobile_sites_smooth'
 189
 190
 191 @benchmark.Disabled('android')  # crbug.com/357326
 192 class PageCyclerToughLayoutCases(_PageCycler):
 193   """Page loading for the slowest layouts observed in the Alexa top 1 million.
 194
 195   Recorded in July 2013.
 196   """
 197   page_set = page_sets.ToughLayoutCasesPageSet
 198
 199   @classmethod
 200   def Name(cls):
 201     return 'page_cycler.tough_layout_cases'
 202
 203
 204 # crbug.com/273986: This test is flakey on Windows.
 205 @benchmark.Disabled('win')
 206 class PageCyclerTypical25(_PageCycler):
 207   """Page load time benchmark for a 25 typical web pages.
 208
 209   Designed to represent typical, not highly optimized or highly popular web
 210   sites. Runs against pages recorded in June, 2014.
 211   """
 212
 213   @classmethod
 214   def Name(cls):
 215     return 'page_cycler.typical_25'
 216
 217   def CreatePageSet(self, options):
 218     return page_sets.Typical25PageSet(run_no_page_interactions=True)
 219
 220 # crbug.com/273986: This test is flakey on Windows.
 221 @benchmark.Disabled  # crbug.com/463346: Test is crashing Chrome.
 222 class PageCyclerOopifTypical25(_PageCycler):
 223   """ A varation of the benchmark above, but running in --site-per-process
 224   to allow measuring performance of out-of-process iframes.
 225   """
 226   @classmethod
 227   def Name(cls):
 228     return 'page_cycler_oopif.typical_25'
 229
 230   def SetExtraBrowserOptions(self, options):
 231     options.AppendExtraBrowserArgs(['--site-per-process'])
 232
 233   def CreatePageSet(self, options):
 234     return page_sets.Typical25PageSet(run_no_page_interactions=True)
 235
 236
 237 @benchmark.Disabled # crbug.com/443730
 238 class PageCyclerBigJs(_PageCycler):
 239   page_set = page_sets.BigJsPageSet
 240   @classmethod
 241   def Name(cls):
 242     return 'page_cycler.big_js'