tools/perf/measurements/measurement_smoke_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 """Measurement smoke test to make sure that no new action_name_to_run is
   5 defined."""
   6
   7 import os
   8 import optparse
   9 import logging
  10 import unittest
  11
  12 from measurements import rasterize_and_record_micro
  13
  14 from telemetry import benchmark as benchmark_module
  15 from telemetry.core import discover
  16 from telemetry.page import page_test
  17 from telemetry.unittest_util import options_for_unittests
  18 from telemetry.util import classes
  19 from telemetry.web_perf import timeline_based_measurement
  20
  21
  22 # Do NOT add new items to this list!
  23 # crbug.com/418375
  24 _ACTION_NAMES_WHITE_LIST = (
  25   'RunPageInteractions',
  26 )
  27
  28
  29 def _GetAllPossiblePageTestInstances():
  30   page_test_instances = []
  31   measurements_dir = os.path.dirname(__file__)
  32   top_level_dir = os.path.dirname(measurements_dir)
  33   benchmarks_dir = os.path.join(top_level_dir, 'benchmarks')
  34
  35   # Get all page test instances from measurement classes that are directly
  36   # constructable
  37   all_measurement_classes = discover.DiscoverClasses(
  38       measurements_dir, top_level_dir, page_test.PageTest).values()
  39   for measurement_class in all_measurement_classes:
  40     if classes.IsDirectlyConstructable(measurement_class):
  41       page_test_instances.append(measurement_class())
  42
  43   all_benchmarks_classes = discover.DiscoverClasses(
  44       benchmarks_dir, top_level_dir, benchmark_module.Benchmark).values()
  45
  46   # Get all page test instances from defined benchmarks.
  47   # Note: since this depends on the command line options, there is no guaranteed
  48   # that this will generate all possible page test instances but it's worth
  49   # enough for smoke test purpose.
  50   for benchmark_class in all_benchmarks_classes:
  51     options = options_for_unittests.GetCopy()
  52     parser = optparse.OptionParser()
  53     benchmark_class.AddCommandLineArgs(parser)
  54     benchmark_module.AddCommandLineArgs(parser)
  55     benchmark_class.SetArgumentDefaults(parser)
  56     options.MergeDefaultValues(parser.get_default_values())
  57     pt = benchmark_class().CreatePageTest(options)
  58     if not isinstance(pt, timeline_based_measurement.TimelineBasedMeasurement):
  59       page_test_instances.append(pt)
  60
  61   return page_test_instances
  62
  63
  64 class MeasurementSmokeTest(unittest.TestCase):
  65   # TODO(nednguyen): Remove this test when crbug.com/418375 is marked fixed.
  66   def testNoNewActionNameToRunUsed(self):
  67     invalid_tests = []
  68     for test in _GetAllPossiblePageTestInstances():
  69       if isinstance(test, rasterize_and_record_micro.RasterizeAndRecordMicro):
  70         continue
  71       if not hasattr(test, 'action_name_to_run'):
  72         invalid_tests.append(test)
  73         logging.error('Test %s missing action_name_to_run attribute.',
  74                       test.__class__.__name__)
  75       if test.action_name_to_run not in _ACTION_NAMES_WHITE_LIST:
  76         invalid_tests.append(test)
  77         logging.error('Page test %s has invalid action_name_to_run: %s' %
  78                      (test.__class__.__name__, repr(test.action_name_to_run)))
  79     self.assertFalse(
  80       invalid_tests,
  81       'New page tests with invalid action_name_to_run found. Please only use '
  82       'action_name_to_run="RunPageInteractions" (crbug.com/418375).')