tools/perf/benchmarks/benchmark_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """For all the benchmarks that set options, test that the options are valid."""
   6
   7 from collections import defaultdict
   8 import os
   9 import unittest
  10
  11 from core import perf_benchmark
  12
  13 from telemetry import benchmark as benchmark_module
  14 from telemetry.core import discover
  15 from telemetry.internal.browser import browser_options
  16 from telemetry.testing import progress_reporter
  17
  18
  19 def _GetPerfDir(*subdirs):
  20   perf_dir = os.path.dirname(os.path.dirname(__file__))
  21   return os.path.join(perf_dir, *subdirs)
  22
  23
  24 def _GetAllPerfBenchmarks():
  25   return discover.DiscoverClasses(
  26       _GetPerfDir('benchmarks'), _GetPerfDir(), benchmark_module.Benchmark,
  27       index_by_class_name=True).values()
  28
  29 def _BenchmarkOptionsTestGenerator(benchmark):
  30   def testBenchmarkOptions(self):  # pylint: disable=W0613
  31     """Invalid options will raise benchmark.InvalidOptionsError."""
  32     options = browser_options.BrowserFinderOptions()
  33     parser = options.CreateParser()
  34     benchmark.AddCommandLineArgs(parser)
  35     benchmark_module.AddCommandLineArgs(parser)
  36     benchmark.SetArgumentDefaults(parser)
  37   return testBenchmarkOptions
  38
  39
  40 class TestNoBenchmarkNamesDuplication(unittest.TestCase):
  41   def runTest(self):
  42     all_benchmarks = _GetAllPerfBenchmarks()
  43     names_to_benchmarks = defaultdict(list)
  44     for b in all_benchmarks:
  45       names_to_benchmarks[b.Name()].append(b)
  46     for n in names_to_benchmarks:
  47       self.assertEquals(1, len(names_to_benchmarks[n]),
  48                         'Multiple benchmarks with the same name %s are '
  49                         'found: %s' % (n, str(names_to_benchmarks[n])))
  50
  51
  52 class TestNoOverrideCustomizeBrowserOptions(unittest.TestCase):
  53   def runTest(self):
  54     all_benchmarks = _GetAllPerfBenchmarks()
  55     for benchmark in all_benchmarks:
  56       self.assertEquals(True, issubclass(benchmark,
  57           perf_benchmark.PerfBenchmark),
  58           'Benchmark %s needs to subclass from PerfBenchmark'
  59           % benchmark.Name())
  60       self.assertEquals(benchmark.CustomizeBrowserOptions,
  61           perf_benchmark.PerfBenchmark.CustomizeBrowserOptions,
  62           'Benchmark %s should not override CustomizeBrowserOptions'
  63           % benchmark.Name())
  64
  65 def _AddBenchmarkOptionsTests(suite):
  66   # Using |index_by_class_name=True| allows returning multiple benchmarks
  67   # from a module.
  68   all_benchmarks = _GetAllPerfBenchmarks()
  69   for benchmark in all_benchmarks:
  70     if not benchmark.options:
  71       # No need to test benchmarks that have not defined options.
  72       continue
  73     class BenchmarkOptionsTest(unittest.TestCase):
  74       pass
  75     setattr(BenchmarkOptionsTest, benchmark.Name(),
  76             _BenchmarkOptionsTestGenerator(benchmark))
  77     suite.addTest(BenchmarkOptionsTest(benchmark.Name()))
  78   suite.addTest(TestNoBenchmarkNamesDuplication())
  79   suite.addTest(TestNoOverrideCustomizeBrowserOptions())
  80
  81
  82 def load_tests(loader, standard_tests, pattern):
  83   del loader, standard_tests, pattern  # unused
  84   suite = progress_reporter.TestSuite()
  85   _AddBenchmarkOptionsTests(suite)
  86   return suite