tools/perf/benchmarks/benchmark_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """For all the benchmarks that set options, test that the options are valid."""
   6
   7 from collections import defaultdict
   8 import logging
   9 import os
  10 import unittest
  11
  12 from telemetry import benchmark as benchmark_module
  13 from telemetry.core import browser_options
  14 from telemetry.core import discover
  15 from telemetry.unittest_util import progress_reporter
  16
  17
  18 def _GetPerfDir(*subdirs):
  19   perf_dir = os.path.dirname(os.path.dirname(__file__))
  20   return os.path.join(perf_dir, *subdirs)
  21
  22
  23 def _GetAllPerfBenchmarks():
  24   return discover.DiscoverClasses(
  25       _GetPerfDir('benchmarks'), _GetPerfDir(), benchmark_module.Benchmark,
  26       index_by_class_name=True).values()
  27
  28 def _BenchmarkOptionsTestGenerator(benchmark):
  29   def testBenchmarkOptions(self):  # pylint: disable=W0613
  30     """Invalid options will raise benchmark.InvalidOptionsError."""
  31     options = browser_options.BrowserFinderOptions()
  32     parser = options.CreateParser()
  33     benchmark.AddCommandLineArgs(parser)
  34     benchmark_module.AddCommandLineArgs(parser)
  35     benchmark.SetArgumentDefaults(parser)
  36   return testBenchmarkOptions
  37
  38
  39 class TestNoBenchmarkNamesDuplication(unittest.TestCase):
  40   def runTest(self):
  41     all_benchmarks = _GetAllPerfBenchmarks()
  42     names_to_benchmarks = defaultdict(list)
  43     for b in all_benchmarks:
  44       names_to_benchmarks[b.Name()].append(b)
  45     for n in names_to_benchmarks:
  46       self.assertEquals(1, len(names_to_benchmarks[n]),
  47                         'Multiple benchmarks with the same name %s are '
  48                         'found: %s' % (n, str(names_to_benchmarks[n])))
  49
  50 def _AddBenchmarkOptionsTests(suite):
  51   # Using |index_by_class_name=True| allows returning multiple benchmarks
  52   # from a module.
  53   all_benchmarks = _GetAllPerfBenchmarks()
  54   for benchmark in all_benchmarks:
  55     if not benchmark.options:
  56       # No need to test benchmarks that have not defined options.
  57       continue
  58     class BenchmarkOptionsTest(unittest.TestCase):
  59       pass
  60     setattr(BenchmarkOptionsTest, benchmark.Name(),
  61             _BenchmarkOptionsTestGenerator(benchmark))
  62     suite.addTest(BenchmarkOptionsTest(benchmark.Name()))
  63   suite.addTest(TestNoBenchmarkNamesDuplication())
  64
  65
  66 def load_tests(_, _2, _3):
  67   suite = progress_reporter.TestSuite()
  68   _AddBenchmarkOptionsTests(suite)
  69   return suite