tools/perf/benchmarks/benchmark_smoke_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Run the first page of one benchmark for every module.
   6
   7 Only benchmarks that have a composable measurement are included.
   8 Ideally this test would be comprehensive, however, running one page
   9 of every benchmark would run impractically long.
  10 """
  11
  12 import os
  13 import sys
  14 import unittest
  15
  16 from telemetry import benchmark as benchmark_module
  17 from telemetry.core import discover
  18 from telemetry.testing import options_for_unittests
  19 from telemetry.testing import progress_reporter
  20
  21 from benchmarks import image_decoding
  22 from benchmarks import indexeddb_perf
  23 from benchmarks import jetstream
  24 from benchmarks import kraken
  25 from benchmarks import memory
  26 from benchmarks import octane
  27 from benchmarks import rasterize_and_record_micro
  28 from benchmarks import repaint
  29 from benchmarks import spaceport
  30 from benchmarks import speedometer
  31 from benchmarks import sunspider
  32 from benchmarks import text_selection
  33
  34
  35 def SmokeTestGenerator(benchmark):
  36   # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  37   #
  38   # This smoke test dynamically tests all benchmarks. So disabling it for one
  39   # failing or flaky benchmark would disable a much wider swath of coverage
  40   # than is usally intended. Instead, if a particular benchmark is failing,
  41   # disable it in tools/perf/benchmarks/*.
  42   @benchmark_module.Disabled('chromeos')  # crbug.com/351114
  43   def BenchmarkSmokeTest(self):
  44     # Only measure a single page so that this test cycles reasonably quickly.
  45     benchmark.options['pageset_repeat'] = 1
  46     benchmark.options['page_repeat'] = 1
  47
  48     class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
  49       def CreateStorySet(self, options):
  50         # pylint: disable=E1002
  51         story_set = super(SinglePageBenchmark, self).CreateStorySet(options)
  52         for story in story_set.stories:
  53           story.skip_waits = True
  54           story_set.stories = [story]
  55           break
  56         return story_set
  57
  58     # Set the benchmark's default arguments.
  59     options = options_for_unittests.GetCopy()
  60     options.output_format = 'none'
  61     options.suppress_gtest_report = True
  62     parser = options.CreateParser()
  63
  64     benchmark.AddCommandLineArgs(parser)
  65     benchmark_module.AddCommandLineArgs(parser)
  66     benchmark.SetArgumentDefaults(parser)
  67     options.MergeDefaultValues(parser.get_default_values())
  68
  69     benchmark.ProcessCommandLineArgs(None, options)
  70     benchmark_module.ProcessCommandLineArgs(None, options)
  71
  72     self.assertEqual(0, SinglePageBenchmark().Run(options),
  73                        msg='Failed: %s' % benchmark)
  74
  75   return BenchmarkSmokeTest
  76
  77
  78 # The list of benchmark modules to be excluded from our smoke tests.
  79 _BLACK_LIST_TEST_MODULES = {
  80     image_decoding, # Always fails on Mac10.9 Tests builder.
  81     indexeddb_perf,  # Always fails on Win7 & Android Tests builder.
  82     octane,  # Often fails & take long time to timeout on cq bot.
  83     rasterize_and_record_micro,  # Always fails on cq bot.
  84     repaint,  # Often fails & takes long time to timeout on cq bot.
  85     spaceport,  # Takes 451 seconds.
  86     speedometer,  # Takes 101 seconds.
  87     jetstream,  # Take 206 seconds.
  88     text_selection, # Always fails on cq bot.
  89     memory  # Flaky on bots, crbug.com/513767
  90 }
  91
  92 # Some smoke benchmark tests that run quickly on desktop platform can be very
  93 # slow on Android. So we create a separate set of black list only for Android.
  94 _ANDROID_BLACK_LIST_MODULES = {
  95     kraken,  # Takes 275 seconds on Android.
  96     sunspider,  # Takes 163 seconds on Android.
  97 }
  98
  99
 100 def load_tests(loader, standard_tests, pattern):
 101   del loader, standard_tests, pattern  # unused
 102   suite = progress_reporter.TestSuite()
 103
 104   benchmarks_dir = os.path.dirname(__file__)
 105   top_level_dir = os.path.dirname(benchmarks_dir)
 106
 107   # Using the default of |index_by_class_name=False| means that if a module
 108   # has multiple benchmarks, only the last one is returned.
 109   all_benchmarks = discover.DiscoverClasses(
 110       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
 111       index_by_class_name=False).values()
 112   for benchmark in all_benchmarks:
 113     if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
 114       continue
 115     # TODO(tonyg): Smoke doesn't work with session_restore yet.
 116     if (benchmark.Name().startswith('session_restore') or
 117         benchmark.Name().startswith('skpicture_printer')):
 118       continue
 119
 120     if hasattr(benchmark, 'generated_profile_archive'):
 121       # We'd like to test these, but don't know how yet.
 122       continue
 123
 124     class BenchmarkSmokeTest(unittest.TestCase):
 125       pass
 126
 127     method = SmokeTestGenerator(benchmark)
 128
 129     # Make sure any decorators are propagated from the original declaration.
 130     # (access to protected members) pylint: disable=W0212
 131     # TODO(dpranke): Since we only pick the first test from every class
 132     # (above), if that test is disabled, we'll end up not running *any*
 133     # test from the class. We should probably discover all of the tests
 134     # in a class, and then throw the ones we don't need away instead.
 135
 136     # Merge decorators.
 137     for attribute in ['_enabled_strings', '_disabled_strings']:
 138       # Do set union of attributes to eliminate duplicates.
 139       merged_attributes = list(set(getattr(method, attribute, []) +
 140                                    getattr(benchmark, attribute, [])))
 141       if merged_attributes:
 142         setattr(method, attribute, merged_attributes)
 143
 144       # Handle the case where the benchmark is Enabled/Disabled everywhere.
 145       if (getattr(method, attribute, None) == [] or
 146           getattr(benchmark, attribute, None) == []):
 147         setattr(method, attribute, [])
 148
 149     # Disable some tests on android platform only.
 150     if sys.modules[benchmark.__module__] in _ANDROID_BLACK_LIST_MODULES:
 151       method._disabled_strings.append('android')
 152
 153     setattr(BenchmarkSmokeTest, benchmark.Name(), method)
 154
 155     suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
 156
 157   return suite