content/test/gpu/gpu_tests/gpu_test_base_unittest.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import os
   5 import unittest
   6
   7 from telemetry import benchmark
   8 from telemetry.core import discover
   9 from telemetry.core import util
  10 from telemetry.story import story_set as story_set_module
  11 from telemetry.testing import fakes
  12
  13 util.AddDirToPythonPath(util.GetTelemetryDir(), 'third_party', 'mock')
  14 import mock # pylint: disable=import-error
  15
  16 import gpu_test_base
  17
  18 def _GetGpuDir(*subdirs):
  19   gpu_dir = os.path.dirname(os.path.dirname(__file__))
  20   return os.path.join(gpu_dir, *subdirs)
  21
  22 # Unit tests verifying invariants of classes in GpuTestBase.
  23
  24 class NoOverridesTest(unittest.TestCase):
  25   def testValidatorBase(self):
  26     all_validators = discover.DiscoverClasses(
  27       _GetGpuDir('gpu_tests'), _GetGpuDir(),
  28       gpu_test_base.ValidatorBase,
  29       index_by_class_name=True).values()
  30     self.assertGreater(len(all_validators), 0)
  31     for validator in all_validators:
  32       self.assertEquals(gpu_test_base.ValidatorBase.ValidateAndMeasurePage,
  33                         validator.ValidateAndMeasurePage,
  34                         'Class %s should not override ValidateAndMeasurePage'
  35                         % validator.__name__)
  36
  37   def testPageBase(self):
  38     all_pages = discover.DiscoverClasses(
  39       _GetGpuDir(), _GetGpuDir(),
  40       gpu_test_base.PageBase,
  41       index_by_class_name=True).values()
  42     self.assertGreater(len(all_pages), 0)
  43     for page in all_pages:
  44       self.assertEquals(gpu_test_base.PageBase.RunNavigateSteps,
  45                         page.RunNavigateSteps,
  46                         'Class %s should not override ValidateAndMeasurePage'
  47                         % page.__name__)
  48       self.assertEquals(gpu_test_base.PageBase.RunPageInteractions,
  49                         page.RunPageInteractions,
  50                         'Class %s should not override RunPageInteractions'
  51                         % page.__name__)
  52
  53 #
  54 # Tests verifying interactions between Telemetry and GpuTestBase.
  55 #
  56
  57 class FakeValidator(gpu_test_base.ValidatorBase):
  58   def __init__(self):
  59     super(FakeValidator, self).__init__()
  60     self.ValidateAndMeasurePageInner = mock.Mock()
  61
  62
  63 class FakePage(gpu_test_base.PageBase):
  64   def __init__(self, page_set, name, expectations=None):
  65     super(FakePage, self).__init__(
  66       name=name,
  67       url='http://nonexistentserver.com/' + name,
  68       page_set=page_set,
  69       shared_page_state_class=fakes.FakeSharedPageState,
  70       expectations=expectations)
  71     self.RunNavigateStepsInner = mock.Mock()
  72     self.RunPageInteractionsInner = mock.Mock()
  73
  74
  75 class FakeTest(gpu_test_base.TestBase):
  76   def __init__(self, max_failures=None):
  77     super(FakeTest, self).__init__(max_failures)
  78     self._fake_pages = []
  79     self._fake_story_set = story_set_module.StorySet()
  80     self._created_story_set = False
  81     self.validator = FakeValidator()
  82
  83   def _CreateExpectations(self):
  84     return super(FakeTest, self)._CreateExpectations()
  85
  86   def CreatePageTest(self, options):
  87     return self.validator
  88
  89   def GetFakeStorySet(self):
  90     return self._fake_story_set
  91
  92   def AddFakePage(self, page):
  93     if self._created_story_set:
  94       raise Exception('Can not add any more fake pages')
  95     self._fake_pages.append(page)
  96
  97   def CreateStorySet(self, options):
  98     if self._created_story_set:
  99       raise Exception('Can only create the story set once per FakeTest')
 100     for page in self._fake_pages:
 101       self._fake_story_set.AddStory(page)
 102     self._created_story_set = True
 103     return self._fake_story_set
 104
 105
 106 class FailingPage(FakePage):
 107   def __init__(self, page_set, name, expectations=None):
 108     super(FailingPage, self).__init__(page_set, name, expectations=expectations)
 109     self.RunNavigateStepsInner.side_effect = Exception('Deliberate exception')
 110
 111
 112 class PageExecutionTest(unittest.TestCase):
 113   def setupTest(self):
 114     finder_options = fakes.CreateBrowserFinderOptions()
 115     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
 116     finder_options.output_formats = ['none']
 117     finder_options.suppress_gtest_report = True
 118     finder_options.output_dir = None
 119     finder_options.upload_bucket = 'public'
 120     finder_options.upload_results = False
 121     testclass = FakeTest
 122     parser = finder_options.CreateParser()
 123     benchmark.AddCommandLineArgs(parser)
 124     testclass.AddCommandLineArgs(parser)
 125     options, dummy_args = parser.parse_args([])
 126     benchmark.ProcessCommandLineArgs(parser, options)
 127     testclass.ProcessCommandLineArgs(parser, options)
 128     test = testclass()
 129     return test, finder_options
 130
 131   def testPassingPage(self):
 132     test, finder_options = self.setupTest()
 133     manager = mock.Mock()
 134     page = FakePage(test.GetFakeStorySet(), 'page1')
 135     page.RunNavigateStepsInner = manager.page.RunNavigateStepsInner
 136     page.RunPageInteractionsInner = manager.page.RunPageInteractionsInner
 137     test.validator.ValidateAndMeasurePageInner = (
 138       manager.validator.ValidateAndMeasurePageInner)
 139     test.AddFakePage(page)
 140     self.assertEqual(test.Run(finder_options), 0,
 141                      'Test should run with no errors')
 142     expected = [mock.call.page.RunNavigateStepsInner(mock.ANY),
 143                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 144                 mock.call.validator.ValidateAndMeasurePageInner(
 145                   page, mock.ANY, mock.ANY)]
 146     self.assertTrue(manager.mock_calls == expected)
 147
 148   def testFailingPage(self):
 149     test, finder_options = self.setupTest()
 150     page = FailingPage(test.GetFakeStorySet(), 'page1')
 151     test.AddFakePage(page)
 152     self.assertNotEqual(test.Run(finder_options), 0, 'Test should fail')
 153
 154   def testExpectedFailure(self):
 155     test, finder_options = self.setupTest()
 156     page = FailingPage(test.GetFakeStorySet(), 'page1',
 157                        expectations=test.GetExpectations())
 158     test.AddFakePage(page)
 159     test.GetExpectations().Fail('page1')
 160     self.assertEqual(test.Run(finder_options), 0,
 161                      'Test should run with no errors')
 162     self.assertFalse(page.RunPageInteractionsInner.called)
 163     self.assertFalse(test.validator.ValidateAndMeasurePageInner.called)
 164
 165   def testPassAfterExpectedFailure(self):
 166     test, finder_options = self.setupTest()
 167     manager = mock.Mock()
 168     page = FailingPage(test.GetFakeStorySet(), 'page1',
 169                        expectations=test.GetExpectations())
 170     test.AddFakePage(page)
 171     test.GetExpectations().Fail('page1')
 172     page = FakePage(test.GetFakeStorySet(), 'page2')
 173     page.RunNavigateStepsInner = manager.page.RunNavigateStepsInner
 174     page.RunPageInteractionsInner = manager.page.RunPageInteractionsInner
 175     test.validator.ValidateAndMeasurePageInner = (
 176       manager.validator.ValidateAndMeasurePageInner)
 177     test.AddFakePage(page)
 178     self.assertEqual(test.Run(finder_options), 0,
 179                      'Test should run with no errors')
 180     expected = [mock.call.page.RunNavigateStepsInner(mock.ANY),
 181                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 182                 mock.call.validator.ValidateAndMeasurePageInner(
 183                   page, mock.ANY, mock.ANY)]
 184     self.assertTrue(manager.mock_calls == expected)