tools/perf/benchmarks/benchmark_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Run the first page of every benchmark that has a composable measurement.
   6
   7 Ideally this test would be comprehensive, but the above serves as a
   8 kind of smoke test.
   9 """
  10
  11 import os
  12 import unittest
  13
  14 from telemetry import benchmark as benchmark_module
  15 from telemetry.core import discover
  16 from telemetry.page import page_test
  17 from telemetry.unittest import options_for_unittests
  18 from telemetry.unittest import progress_reporter
  19
  20
  21 def SmokeTestGenerator(benchmark):
  22   # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  23   #
  24   # This smoke test dynamically tests all benchmarks. So disabling it for one
  25   # failing or flaky benchmark would disable a much wider swath of coverage
  26   # than is usally intended. Instead, if a particular benchmark is failing,
  27   # disable it in tools/perf/benchmarks/*.
  28   @benchmark_module.Disabled('chromeos')  # crbug.com/351114
  29   def BenchmarkSmokeTest(self):
  30     # Only measure a single page so that this test cycles reasonably quickly.
  31     benchmark.options['pageset_repeat'] = 1
  32     benchmark.options['page_repeat'] = 1
  33
  34     class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
  35       def CreatePageSet(self, options):
  36         # pylint: disable=E1002
  37         ps = super(SinglePageBenchmark, self).CreatePageSet(options)
  38         for p in ps.pages:
  39           if not p.disabled:
  40             p.skip_waits = True
  41             ps.pages = [p]
  42             break
  43         return ps
  44
  45     # Set the benchmark's default arguments.
  46     options = options_for_unittests.GetCopy()
  47     options.output_format = 'none'
  48     options.suppress_gtest_report = True
  49     parser = options.CreateParser()
  50
  51     benchmark.AddCommandLineArgs(parser)
  52     benchmark_module.AddCommandLineArgs(parser)
  53     benchmark.SetArgumentDefaults(parser)
  54     options.MergeDefaultValues(parser.get_default_values())
  55
  56     benchmark.ProcessCommandLineArgs(None, options)
  57     benchmark_module.ProcessCommandLineArgs(None, options)
  58
  59     self.assertEqual(0, SinglePageBenchmark().Run(options),
  60                      msg='Failed: %s' % benchmark)
  61
  62   return BenchmarkSmokeTest
  63
  64
  65 def load_tests(_, _2, _3):
  66   suite = progress_reporter.TestSuite()
  67
  68   benchmarks_dir = os.path.dirname(__file__)
  69   top_level_dir = os.path.dirname(benchmarks_dir)
  70   measurements_dir = os.path.join(top_level_dir, 'measurements')
  71
  72   all_measurements = discover.DiscoverClasses(
  73       measurements_dir, top_level_dir, page_test.PageTest,
  74       pattern='*.py').values()
  75   all_benchmarks = discover.DiscoverClasses(
  76       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
  77       pattern='*.py').values()
  78   for benchmark in all_benchmarks:
  79     if benchmark.PageTestClass() not in all_measurements:
  80       # If the benchmark is not in measurements, then it is not composable.
  81       # Ideally we'd like to test these as well, but the non-composable
  82       # benchmarks are usually long-running benchmarks.
  83       continue
  84
  85     # TODO(tonyg): Smoke doesn't work with session_restore yet.
  86     if benchmark.Name().startswith('session_restore'):
  87       continue
  88
  89     if hasattr(benchmark, 'generated_profile_archive'):
  90       # We'd like to test these, but don't know how yet.
  91       continue
  92
  93     class BenchmarkSmokeTest(unittest.TestCase):
  94       pass
  95     setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark))
  96     suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
  97
  98   return suite