tools/perf/benchmarks/benchmark_smoke_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Run the first page of one benchmark for every module.
   6
   7 Only benchmarks that have a composable measurement are included.
   8 Ideally this test would be comprehensive, however, running one page
   9 of every benchmark would run impractically long.
  10 """
  11
  12 import os
  13 import unittest
  14
  15 from telemetry import benchmark as benchmark_module
  16 from telemetry.core import discover
  17 from telemetry.page import page_test
  18 from telemetry.unittest_util import options_for_unittests
  19 from telemetry.unittest_util import progress_reporter
  20
  21
  22 def SmokeTestGenerator(benchmark):
  23   # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  24   #
  25   # This smoke test dynamically tests all benchmarks. So disabling it for one
  26   # failing or flaky benchmark would disable a much wider swath of coverage
  27   # than is usally intended. Instead, if a particular benchmark is failing,
  28   # disable it in tools/perf/benchmarks/*.
  29
  30   # EXCEPTIONS TO THE ABOVE RULE: crbug.com/351114, crbug.com/423688
  31   @benchmark_module.Disabled('chromeos', 'mavericks')
  32   def BenchmarkSmokeTest(self):
  33     # Only measure a single page so that this test cycles reasonably quickly.
  34     benchmark.options['pageset_repeat'] = 1
  35     benchmark.options['page_repeat'] = 1
  36
  37     class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
  38       def CreatePageSet(self, options):
  39         # pylint: disable=E1002
  40         ps = super(SinglePageBenchmark, self).CreatePageSet(options)
  41         for p in ps.pages:
  42           p.skip_waits = True
  43           ps.user_stories = [p]
  44           break
  45         return ps
  46
  47     # Set the benchmark's default arguments.
  48     options = options_for_unittests.GetCopy()
  49     options.output_format = 'none'
  50     options.suppress_gtest_report = True
  51     parser = options.CreateParser()
  52
  53     benchmark.AddCommandLineArgs(parser)
  54     benchmark_module.AddCommandLineArgs(parser)
  55     benchmark.SetArgumentDefaults(parser)
  56     options.MergeDefaultValues(parser.get_default_values())
  57
  58     benchmark.ProcessCommandLineArgs(None, options)
  59     benchmark_module.ProcessCommandLineArgs(None, options)
  60
  61     self.assertEqual(0, SinglePageBenchmark().Run(options),
  62                      msg='Failed: %s' % benchmark)
  63
  64   return BenchmarkSmokeTest
  65
  66
  67 def load_tests(_, _2, _3):
  68   suite = progress_reporter.TestSuite()
  69
  70   benchmarks_dir = os.path.dirname(__file__)
  71   top_level_dir = os.path.dirname(benchmarks_dir)
  72   measurements_dir = os.path.join(top_level_dir, 'measurements')
  73
  74   all_measurements = discover.DiscoverClasses(
  75       measurements_dir, top_level_dir, page_test.PageTest).values()
  76   # Using the default of |index_by_class_name=False| means that if a module
  77   # has multiple benchmarks, only the last one is returned.
  78   all_benchmarks = discover.DiscoverClasses(
  79       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
  80       index_by_class_name=False).values()
  81   for benchmark in all_benchmarks:
  82     if hasattr(benchmark, 'test') and benchmark.test not in all_measurements:
  83       # If the benchmark does not have a measurement, then it is not composable.
  84       # Ideally we'd like to test these as well, but the non-composable
  85       # benchmarks are usually long-running benchmarks.
  86       continue
  87
  88     # TODO(tonyg): Smoke doesn't work with session_restore yet.
  89     if (benchmark.Name().startswith('session_restore') or
  90         benchmark.Name().startswith('skpicture_printer')):
  91       continue
  92
  93     if hasattr(benchmark, 'generated_profile_archive'):
  94       # We'd like to test these, but don't know how yet.
  95       continue
  96
  97     class BenchmarkSmokeTest(unittest.TestCase):
  98       pass
  99
 100     method = SmokeTestGenerator(benchmark)
 101
 102     # Make sure any decorators are propagated from the original declaration.
 103     # (access to protected members) pylint: disable=W0212
 104     # TODO(dpranke): Since we only pick the first test from every class
 105     # (above), if that test is disabled, we'll end up not running *any*
 106     # test from the class. We should probably discover all of the tests
 107     # in a class, and then throw the ones we don't need away instead.
 108     if hasattr(benchmark, '_enabled_strings'):
 109       method._enabled_strings = benchmark._enabled_strings
 110     if hasattr(benchmark, '_disabled_strings'):
 111       method._disabled_strings = benchmark._disabled_strings
 112     setattr(BenchmarkSmokeTest, benchmark.Name(), method)
 113
 114     suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
 115
 116   return suite