content/test/gpu/gpu_tests/gpu_test_base_unittest.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import os
   5 import unittest
   6
   7 from telemetry import benchmark
   8 from telemetry.core import discover
   9 from telemetry.core import exceptions
  10 from telemetry.core import util
  11 from telemetry.story import story_set as story_set_module
  12 from telemetry.testing import fakes
  13
  14 import mock # pylint: disable=import-error
  15
  16 import gpu_test_base
  17
  18 def _GetGpuDir(*subdirs):
  19   gpu_dir = os.path.dirname(os.path.dirname(__file__))
  20   return os.path.join(gpu_dir, *subdirs)
  21
  22 # Unit tests verifying invariants of classes in GpuTestBase.
  23
  24 class NoOverridesTest(unittest.TestCase):
  25   def testValidatorBase(self):
  26     all_validators = discover.DiscoverClasses(
  27       _GetGpuDir('gpu_tests'), _GetGpuDir(),
  28       gpu_test_base.ValidatorBase,
  29       index_by_class_name=True).values()
  30     self.assertGreater(len(all_validators), 0)
  31     for validator in all_validators:
  32       self.assertEquals(gpu_test_base.ValidatorBase.WillNavigateToPage,
  33                         validator.WillNavigateToPage,
  34                         'Class %s should not override WillNavigateToPage'
  35                         % validator.__name__)
  36       self.assertEquals(gpu_test_base.ValidatorBase.DidNavigateToPage,
  37                         validator.DidNavigateToPage,
  38                         'Class %s should not override DidNavigateToPage'
  39                         % validator.__name__)
  40       self.assertEquals(gpu_test_base.ValidatorBase.ValidateAndMeasurePage,
  41                         validator.ValidateAndMeasurePage,
  42                         'Class %s should not override ValidateAndMeasurePage'
  43                         % validator.__name__)
  44
  45   def testPageBase(self):
  46     all_pages = discover.DiscoverClasses(
  47       _GetGpuDir(), _GetGpuDir(),
  48       gpu_test_base.PageBase,
  49       index_by_class_name=True).values()
  50     self.assertGreater(len(all_pages), 0)
  51     for page in all_pages:
  52       self.assertEquals(gpu_test_base.PageBase.RunNavigateSteps,
  53                         page.RunNavigateSteps,
  54                         'Class %s should not override RunNavigateSteps'
  55                         % page.__name__)
  56       self.assertEquals(gpu_test_base.PageBase.RunPageInteractions,
  57                         page.RunPageInteractions,
  58                         'Class %s should not override RunPageInteractions'
  59                         % page.__name__)
  60
  61 #
  62 # Tests verifying interactions between Telemetry and GpuTestBase.
  63 #
  64
  65 class FakeValidator(gpu_test_base.ValidatorBase):
  66   def __init__(self, manager_mock=None):
  67     super(FakeValidator, self).__init__()
  68     if manager_mock == None:
  69       self.WillNavigateToPageInner = mock.Mock()
  70       self.DidNavigateToPageInner = mock.Mock()
  71       self.ValidateAndMeasurePageInner = mock.Mock()
  72     else:
  73       self.WillNavigateToPageInner = manager_mock.WillNavigateToPageInner
  74       self.DidNavigateToPageInner = manager_mock.DidNavigateToPageInner
  75       self.ValidateAndMeasurePageInner = \
  76         manager_mock.ValidateAndMeasurePageInner
  77
  78
  79 class FakeGpuSharedPageState(fakes.FakeSharedPageState):
  80   # NOTE: if you change this logic you must change the logic in
  81   # GpuSharedPageState (gpu_test_base.py) as well.
  82   def CanRunOnBrowser(self, browser_info, page):
  83     expectations = page.GetExpectations()
  84     return expectations.GetExpectationForPage(
  85       browser_info.browser, page) != 'skip'
  86
  87
  88 class FakePage(gpu_test_base.PageBase):
  89   def __init__(self, benchmark, name, manager_mock=None):
  90     super(FakePage, self).__init__(
  91       name=name,
  92       url='http://nonexistentserver.com/' + name,
  93       page_set=benchmark.GetFakeStorySet(),
  94       shared_page_state_class=FakeGpuSharedPageState,
  95       expectations=benchmark.GetExpectations())
  96     if manager_mock == None:
  97       self.RunNavigateStepsInner = mock.Mock()
  98       self.RunPageInteractionsInner = mock.Mock()
  99     else:
 100       self.RunNavigateStepsInner = manager_mock.RunNavigateStepsInner
 101       self.RunPageInteractionsInner = manager_mock.RunPageInteractionsInner
 102
 103
 104 class FakeTest(gpu_test_base.TestBase):
 105   def __init__(self, manager_mock=None, max_failures=None):
 106     super(FakeTest, self).__init__(max_failures)
 107     self._fake_pages = []
 108     self._fake_story_set = story_set_module.StorySet()
 109     self._created_story_set = False
 110     validator_mock = manager_mock.validator if manager_mock else None
 111     self.validator = FakeValidator(manager_mock=validator_mock)
 112
 113   def _CreateExpectations(self):
 114     return super(FakeTest, self)._CreateExpectations()
 115
 116   def CreatePageTest(self, options):
 117     return self.validator
 118
 119   def GetFakeStorySet(self):
 120     return self._fake_story_set
 121
 122   def AddFakePage(self, page):
 123     if self._created_story_set:
 124       raise Exception('Can not add any more fake pages')
 125     self._fake_pages.append(page)
 126
 127   def CreateStorySet(self, options):
 128     if self._created_story_set:
 129       raise Exception('Can only create the story set once per FakeTest')
 130     for page in self._fake_pages:
 131       self._fake_story_set.AddStory(page)
 132     self._created_story_set = True
 133     return self._fake_story_set
 134
 135
 136 class FailingPage(FakePage):
 137   def __init__(self, benchmark, name, manager_mock=None):
 138     super(FailingPage, self).__init__(benchmark, name,
 139                                       manager_mock=manager_mock)
 140     self.RunNavigateStepsInner.side_effect = Exception('Deliberate exception')
 141
 142
 143 class CrashingPage(FakePage):
 144   def __init__(self, benchmark, name, manager_mock=None):
 145     super(CrashingPage, self).__init__(benchmark, name,
 146                                        manager_mock=manager_mock)
 147     self.RunNavigateStepsInner.side_effect = (
 148       exceptions.DevtoolsTargetCrashException(None))
 149
 150
 151 class PageWhichFailsNTimes(FakePage):
 152   def __init__(self, benchmark, name, times_to_fail, manager_mock=None):
 153     super(PageWhichFailsNTimes, self).__init__(benchmark, name,
 154                                                manager_mock=manager_mock)
 155     self._times_to_fail = times_to_fail
 156     self.RunNavigateStepsInner.side_effect = self.maybeFail
 157
 158   def maybeFail(self, action_runner):
 159     if self._times_to_fail > 0:
 160       self._times_to_fail = self._times_to_fail - 1
 161       raise Exception('Deliberate exception')
 162
 163 class PageRunExecutionTest(unittest.TestCase):
 164   def testNoGarbageCollectionCalls(self):
 165     mock_shared_state = mock.Mock()
 166     p = gpu_test_base.PageBase('file://foo.html')
 167     p.Run(mock_shared_state)
 168     expected = [mock.call.page_test.WillNavigateToPage(
 169                   p, mock_shared_state.current_tab),
 170                 mock.call.page_test.RunNavigateSteps(
 171                   p, mock_shared_state.current_tab),
 172                 mock.call.page_test.DidNavigateToPage(
 173                   p, mock_shared_state.current_tab)]
 174     self.assertEquals(mock_shared_state.mock_calls, expected)
 175
 176 class PageExecutionTest(unittest.TestCase):
 177   def setupTest(self, manager_mock=None):
 178     finder_options = fakes.CreateBrowserFinderOptions()
 179     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
 180     finder_options.output_formats = ['none']
 181     finder_options.suppress_gtest_report = True
 182     finder_options.output_dir = None
 183     finder_options.upload_bucket = 'public'
 184     finder_options.upload_results = False
 185     testclass = FakeTest
 186     parser = finder_options.CreateParser()
 187     benchmark.AddCommandLineArgs(parser)
 188     testclass.AddCommandLineArgs(parser)
 189     options, dummy_args = parser.parse_args([])
 190     benchmark.ProcessCommandLineArgs(parser, options)
 191     testclass.ProcessCommandLineArgs(parser, options)
 192     test = testclass(manager_mock=manager_mock)
 193     return test, finder_options
 194
 195   # Test page.Run() method is called by telemetry framework before
 196   # ValidateAndMeasurePageInner.
 197   def testPageRunMethodIsCalledBeforeValidateAndMeasurePage(self):
 198     manager = mock.Mock()
 199     test, finder_options = self.setupTest(manager)
 200     page = FakePage(test, 'page1')
 201     page.Run = manager.Run
 202     test.AddFakePage(page)
 203     self.assertEqual(test.Run(finder_options), 0,
 204                      'Test should run with no errors')
 205     expected = [mock.call.Run(mock.ANY),
 206                 mock.call.validator.ValidateAndMeasurePageInner(
 207                   page, mock.ANY, mock.ANY)]
 208     self.assertEquals(manager.mock_calls, expected)
 209
 210   def testPassingPage(self):
 211     manager = mock.Mock()
 212     test, finder_options = self.setupTest(manager_mock=manager)
 213     page = FakePage(test, 'page1', manager_mock=manager.page)
 214     test.AddFakePage(page)
 215     self.assertEqual(test.Run(finder_options), 0,
 216                      'Test should run with no errors')
 217     expected = [mock.call.validator.WillNavigateToPageInner(
 218                   page, mock.ANY),
 219                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 220                 mock.call.validator.DidNavigateToPageInner(
 221                   page, mock.ANY),
 222                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 223                 mock.call.validator.ValidateAndMeasurePageInner(
 224                   page, mock.ANY, mock.ANY)]
 225     self.assertTrue(manager.mock_calls == expected)
 226
 227   def testFailingPage(self):
 228     test, finder_options = self.setupTest()
 229     page = FailingPage(test, 'page1')
 230     test.AddFakePage(page)
 231     self.assertNotEqual(test.Run(finder_options), 0, 'Test should fail')
 232
 233   def testExpectedFailure(self):
 234     test, finder_options = self.setupTest()
 235     page = FailingPage(test, 'page1')
 236     test.AddFakePage(page)
 237     test.GetExpectations().Fail('page1')
 238     self.assertEqual(test.Run(finder_options), 0,
 239                      'Test should run with no errors')
 240     self.assertFalse(page.RunPageInteractionsInner.called)
 241     self.assertFalse(test.validator.ValidateAndMeasurePageInner.called)
 242
 243   def testPageSetRepeatOfPageWhichFailsOnce(self):
 244     test, finder_options = self.setupTest()
 245     finder_options.pageset_repeat = 2
 246     page = PageWhichFailsNTimes(test, 'page1', 1)
 247     test.AddFakePage(page)
 248     test.GetExpectations().Fail('page1')
 249     self.assertEqual(test.Run(finder_options), 0,
 250                      'Test should run with no errors')
 251     # This will be called the second time through the page set, when
 252     # the page doesn't fail.
 253     self.assertTrue(page.RunPageInteractionsInner.called)
 254
 255   def testSkipAtPageBaseLevel(self):
 256     test, finder_options = self.setupTest()
 257     page = FailingPage(test, 'page1')
 258     test.AddFakePage(page)
 259     test.GetExpectations().Skip('page1')
 260     self.assertEqual(test.Run(finder_options), 0,
 261                      'Test should run with no errors')
 262     self.assertFalse(test.validator.WillNavigateToPageInner.called)
 263     self.assertFalse(page.RunNavigateStepsInner.called)
 264     self.assertFalse(test.validator.DidNavigateToPageInner.called)
 265     self.assertFalse(page.RunPageInteractionsInner.called)
 266     self.assertFalse(test.validator.ValidateAndMeasurePageInner.called)
 267
 268   def testSkipAtPageLevel(self):
 269     test, finder_options = self.setupTest()
 270     page = FakePage(test, 'page1')
 271     page.RunNavigateSteps = mock.Mock()
 272     page.RunPageInteractions = mock.Mock()
 273     test.validator.WillNavigateToPage = mock.Mock()
 274     test.validator.DidNavigateToPage = mock.Mock()
 275     test.validator.ValidateAndMeasurePage = mock.Mock()
 276     test.AddFakePage(page)
 277     test.GetExpectations().Skip('page1')
 278     self.assertEqual(test.Run(finder_options), 0,
 279                      'Test should run with no errors')
 280     self.assertFalse(test.validator.WillNavigateToPage.called)
 281     self.assertFalse(page.RunNavigateSteps.called)
 282     self.assertFalse(test.validator.DidNavigateToPage.called)
 283     self.assertFalse(page.RunPageInteractions.called)
 284     self.assertFalse(test.validator.ValidateAndMeasurePage.called)
 285
 286   def testPassAfterExpectedFailure(self):
 287     manager = mock.Mock()
 288     test, finder_options = self.setupTest(manager_mock=manager)
 289     page1 = FailingPage(test, 'page1', manager_mock=manager.page1)
 290     test.AddFakePage(page1)
 291     test.GetExpectations().Fail('page1')
 292     page2 = FakePage(test, 'page2', manager_mock=manager.page2)
 293     test.AddFakePage(page2)
 294     self.assertEqual(test.Run(finder_options), 0,
 295                      'Test should run with no errors')
 296     expected = [mock.call.validator.WillNavigateToPageInner(
 297                   page1, mock.ANY),
 298                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 299                 mock.call.validator.WillNavigateToPageInner(
 300                   page2, mock.ANY),
 301                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 302                 mock.call.validator.DidNavigateToPageInner(
 303                   page2, mock.ANY),
 304                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 305                 mock.call.validator.ValidateAndMeasurePageInner(
 306                   page2, mock.ANY, mock.ANY)]
 307     self.assertTrue(manager.mock_calls == expected)
 308
 309   def testExpectedDevtoolsTargetCrash(self):
 310     manager = mock.Mock()
 311     test, finder_options = self.setupTest(manager_mock=manager)
 312     page = CrashingPage(test, 'page1', manager_mock=manager.page)
 313     test.AddFakePage(page)
 314     test.GetExpectations().Fail('page1')
 315     self.assertEqual(test.Run(finder_options), 0,
 316                      'Test should run with no errors')
 317     expected = [mock.call.validator.WillNavigateToPageInner(
 318                   page, mock.ANY),
 319                 mock.call.page.RunNavigateStepsInner(mock.ANY)]
 320     self.assertTrue(manager.mock_calls == expected)
 321
 322   def testFlakyPage(self):
 323     manager = mock.Mock()
 324     test, finder_options = self.setupTest(manager_mock=manager)
 325     page = PageWhichFailsNTimes(test, 'page1', 1, manager_mock=manager.page)
 326     test.AddFakePage(page)
 327     test.GetExpectations().Flaky('page1')
 328     self.assertEqual(test.Run(finder_options), 0,
 329                      'Test should run with no errors')
 330     expected = [mock.call.validator.WillNavigateToPageInner(
 331                   page, mock.ANY),
 332                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 333                 mock.call.validator.WillNavigateToPageInner(
 334                   page, mock.ANY),
 335                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 336                 mock.call.validator.DidNavigateToPageInner(
 337                   page, mock.ANY),
 338                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 339                 mock.call.validator.ValidateAndMeasurePageInner(
 340                   page, mock.ANY, mock.ANY)]
 341     self.assertTrue(manager.mock_calls == expected)
 342
 343   def testFlakyPageExceedingNumRetries(self):
 344     manager = mock.Mock()
 345     test, finder_options = self.setupTest(manager_mock=manager)
 346     page = PageWhichFailsNTimes(test, 'page1', 2, manager_mock=manager.page)
 347     test.AddFakePage(page)
 348     test.GetExpectations().Flaky('page1', max_num_retries=1)
 349     self.assertNotEqual(test.Run(finder_options), 0,
 350                      'Test should fail')
 351     expected = [mock.call.validator.WillNavigateToPageInner(
 352                   page, mock.ANY),
 353                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 354                 mock.call.validator.WillNavigateToPageInner(
 355                   page, mock.ANY),
 356                 mock.call.page.RunNavigateStepsInner(mock.ANY)]
 357     self.assertTrue(manager.mock_calls == expected)
 358
 359   def testFlakyPageThenPassingPage(self):
 360     manager = mock.Mock()
 361     test, finder_options = self.setupTest(manager_mock=manager)
 362     page1 = PageWhichFailsNTimes(test, 'page1', 1, manager_mock=manager.page1)
 363     test.AddFakePage(page1)
 364     page2 = FakePage(test, 'page2', manager_mock=manager.page2)
 365     test.AddFakePage(page2)
 366     test.GetExpectations().Flaky('page1', max_num_retries=1)
 367     self.assertEqual(test.Run(finder_options), 0,
 368                      'Test should run with no errors')
 369     expected = [mock.call.validator.WillNavigateToPageInner(
 370                   page1, mock.ANY),
 371                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 372                 mock.call.validator.WillNavigateToPageInner(
 373                   page1, mock.ANY),
 374                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 375                 mock.call.validator.DidNavigateToPageInner(
 376                   page1, mock.ANY),
 377                 mock.call.page1.RunPageInteractionsInner(mock.ANY),
 378                 mock.call.validator.ValidateAndMeasurePageInner(
 379                   page1, mock.ANY, mock.ANY),
 380                 mock.call.validator.WillNavigateToPageInner(
 381                   page2, mock.ANY),
 382                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 383                 mock.call.validator.DidNavigateToPageInner(
 384                   page2, mock.ANY),
 385                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 386                 mock.call.validator.ValidateAndMeasurePageInner(
 387                   page2, mock.ANY, mock.ANY)]
 388     self.assertTrue(manager.mock_calls == expected)
 389
 390   def testPassingPageThenFlakyPage(self):
 391     manager = mock.Mock()
 392     test, finder_options = self.setupTest(manager_mock=manager)
 393     page1 = FakePage(test, 'page1', manager_mock=manager.page1)
 394     test.AddFakePage(page1)
 395     page2 = PageWhichFailsNTimes(test, 'page2', 1, manager_mock=manager.page2)
 396     test.AddFakePage(page2)
 397     test.GetExpectations().Flaky('page2', max_num_retries=1)
 398     self.assertEqual(test.Run(finder_options), 0,
 399                      'Test should run with no errors')
 400     expected = [mock.call.validator.WillNavigateToPageInner(
 401                   page1, mock.ANY),
 402                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 403                 mock.call.validator.DidNavigateToPageInner(
 404                   page1, mock.ANY),
 405                 mock.call.page1.RunPageInteractionsInner(mock.ANY),
 406                 mock.call.validator.ValidateAndMeasurePageInner(
 407                   page1, mock.ANY, mock.ANY),
 408                 mock.call.validator.WillNavigateToPageInner(
 409                   page2, mock.ANY),
 410                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 411                 mock.call.validator.WillNavigateToPageInner(
 412                   page2, mock.ANY),
 413                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 414                 mock.call.validator.DidNavigateToPageInner(
 415                   page2, mock.ANY),
 416                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 417                 mock.call.validator.ValidateAndMeasurePageInner(
 418                   page2, mock.ANY, mock.ANY)]
 419     self.assertTrue(manager.mock_calls == expected)