content/test/gpu/gpu_tests/gpu_test_base_unittest.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import os
   5 import unittest
   6
   7 from telemetry import benchmark
   8 from telemetry.core import discover
   9 from telemetry.core import exceptions
  10 from telemetry.core import util
  11 from telemetry.story import story_set as story_set_module
  12 from telemetry.testing import fakes
  13
  14 import mock # pylint: disable=import-error
  15
  16 import gpu_test_base
  17
  18 def _GetGpuDir(*subdirs):
  19   gpu_dir = os.path.dirname(os.path.dirname(__file__))
  20   return os.path.join(gpu_dir, *subdirs)
  21
  22 # Unit tests verifying invariants of classes in GpuTestBase.
  23
  24 class NoOverridesTest(unittest.TestCase):
  25   def testValidatorBase(self):
  26     all_validators = discover.DiscoverClasses(
  27       _GetGpuDir('gpu_tests'), _GetGpuDir(),
  28       gpu_test_base.ValidatorBase,
  29       index_by_class_name=True).values()
  30     self.assertGreater(len(all_validators), 0)
  31     for validator in all_validators:
  32       self.assertEquals(gpu_test_base.ValidatorBase.WillNavigateToPage,
  33                         validator.WillNavigateToPage,
  34                         'Class %s should not override WillNavigateToPage'
  35                         % validator.__name__)
  36       self.assertEquals(gpu_test_base.ValidatorBase.DidNavigateToPage,
  37                         validator.DidNavigateToPage,
  38                         'Class %s should not override DidNavigateToPage'
  39                         % validator.__name__)
  40       self.assertEquals(gpu_test_base.ValidatorBase.ValidateAndMeasurePage,
  41                         validator.ValidateAndMeasurePage,
  42                         'Class %s should not override ValidateAndMeasurePage'
  43                         % validator.__name__)
  44
  45   def testPageBase(self):
  46     all_pages = discover.DiscoverClasses(
  47       _GetGpuDir(), _GetGpuDir(),
  48       gpu_test_base.PageBase,
  49       index_by_class_name=True).values()
  50     self.assertGreater(len(all_pages), 0)
  51     for page in all_pages:
  52       self.assertEquals(gpu_test_base.PageBase.RunNavigateSteps,
  53                         page.RunNavigateSteps,
  54                         'Class %s should not override RunNavigateSteps'
  55                         % page.__name__)
  56       self.assertEquals(gpu_test_base.PageBase.RunPageInteractions,
  57                         page.RunPageInteractions,
  58                         'Class %s should not override RunPageInteractions'
  59                         % page.__name__)
  60
  61 #
  62 # Tests verifying interactions between Telemetry and GpuTestBase.
  63 #
  64
  65 class FakeValidator(gpu_test_base.ValidatorBase):
  66   def __init__(self, manager_mock=None):
  67     super(FakeValidator, self).__init__()
  68     if manager_mock == None:
  69       self.WillNavigateToPageInner = mock.Mock()
  70       self.DidNavigateToPageInner = mock.Mock()
  71       self.ValidateAndMeasurePageInner = mock.Mock()
  72     else:
  73       self.WillNavigateToPageInner = manager_mock.WillNavigateToPageInner
  74       self.DidNavigateToPageInner = manager_mock.DidNavigateToPageInner
  75       self.ValidateAndMeasurePageInner = \
  76         manager_mock.ValidateAndMeasurePageInner
  77
  78
  79 class ValidatorWhichFailsNTimes(FakeValidator):
  80   def __init__(self, times_to_fail=0, manager_mock=None):
  81     super(ValidatorWhichFailsNTimes, self).__init__(manager_mock=manager_mock)
  82     self._times_to_fail = times_to_fail
  83     self.ValidateAndMeasurePageInner.side_effect = self.maybeFail
  84
  85   def maybeFail(self, page, tab, results):
  86     if self._times_to_fail > 0:
  87       self._times_to_fail = self._times_to_fail - 1
  88       raise Exception('Deliberate exception')
  89
  90
  91 class FakeGpuSharedPageState(fakes.FakeSharedPageState):
  92   # NOTE: if you change this logic you must change the logic in
  93   # GpuSharedPageState (gpu_test_base.py) as well.
  94   def CanRunOnBrowser(self, browser_info, page):
  95     expectations = page.GetExpectations()
  96     return expectations.GetExpectationForPage(
  97       browser_info.browser, page) != 'skip'
  98
  99   # NOTE: if you change this logic you must change the logic in
 100   # GpuSharedPageState and DesktopGpuSharedPageState
 101   # (gpu_test_base.py) as well.
 102   def RunStory(self, results):
 103     gpu_test_base.RunStoryWithRetries(FakeGpuSharedPageState, self, results)
 104
 105
 106 class FakePage(gpu_test_base.PageBase):
 107   def __init__(self, benchmark, name, manager_mock=None):
 108     super(FakePage, self).__init__(
 109       name=name,
 110       url='http://nonexistentserver.com/' + name,
 111       page_set=benchmark.GetFakeStorySet(),
 112       shared_page_state_class=FakeGpuSharedPageState,
 113       expectations=benchmark.GetExpectations())
 114     if manager_mock == None:
 115       self.RunNavigateStepsInner = mock.Mock()
 116       self.RunPageInteractionsInner = mock.Mock()
 117     else:
 118       self.RunNavigateStepsInner = manager_mock.RunNavigateStepsInner
 119       self.RunPageInteractionsInner = manager_mock.RunPageInteractionsInner
 120
 121
 122 class FakeTest(gpu_test_base.TestBase):
 123   def __init__(self,
 124                times_to_fail_test=0,
 125                manager_mock=None,
 126                max_failures=None):
 127     super(FakeTest, self).__init__(max_failures)
 128     self._fake_pages = []
 129     self._fake_story_set = story_set_module.StorySet()
 130     self._created_story_set = False
 131     validator_mock = manager_mock.validator if manager_mock else None
 132     self.validator = FakeValidator(manager_mock=validator_mock)
 133     if times_to_fail_test > 0:
 134       self.validator = ValidatorWhichFailsNTimes(
 135         times_to_fail=times_to_fail_test,
 136         manager_mock=validator_mock)
 137     else:
 138       self.validator = FakeValidator(manager_mock=validator_mock)
 139
 140   def _CreateExpectations(self):
 141     return super(FakeTest, self)._CreateExpectations()
 142
 143   def CreatePageTest(self, options):
 144     return self.validator
 145
 146   def GetFakeStorySet(self):
 147     return self._fake_story_set
 148
 149   def AddFakePage(self, page):
 150     if self._created_story_set:
 151       raise Exception('Can not add any more fake pages')
 152     self._fake_pages.append(page)
 153
 154   def CreateStorySet(self, options):
 155     if self._created_story_set:
 156       raise Exception('Can only create the story set once per FakeTest')
 157     for page in self._fake_pages:
 158       self._fake_story_set.AddStory(page)
 159     self._created_story_set = True
 160     return self._fake_story_set
 161
 162
 163 class FailingPage(FakePage):
 164   def __init__(self, benchmark, name, manager_mock=None):
 165     super(FailingPage, self).__init__(benchmark, name,
 166                                       manager_mock=manager_mock)
 167     self.RunNavigateStepsInner.side_effect = Exception('Deliberate exception')
 168
 169
 170 class CrashingPage(FakePage):
 171   def __init__(self, benchmark, name, manager_mock=None):
 172     super(CrashingPage, self).__init__(benchmark, name,
 173                                        manager_mock=manager_mock)
 174     self.RunNavigateStepsInner.side_effect = (
 175       exceptions.DevtoolsTargetCrashException(None))
 176
 177
 178 class PageWhichFailsNTimes(FakePage):
 179   def __init__(self, benchmark, name, times_to_fail, manager_mock=None):
 180     super(PageWhichFailsNTimes, self).__init__(benchmark, name,
 181                                                manager_mock=manager_mock)
 182     self._times_to_fail = times_to_fail
 183     self.RunNavigateStepsInner.side_effect = self.maybeFail
 184
 185   def maybeFail(self, action_runner):
 186     if self._times_to_fail > 0:
 187       self._times_to_fail = self._times_to_fail - 1
 188       raise Exception('Deliberate exception')
 189
 190 class PageRunExecutionTest(unittest.TestCase):
 191   def testNoGarbageCollectionCalls(self):
 192     mock_shared_state = mock.Mock()
 193     p = gpu_test_base.PageBase('file://foo.html')
 194     p.Run(mock_shared_state)
 195     expected = [mock.call.page_test.WillNavigateToPage(
 196                   p, mock_shared_state.current_tab),
 197                 mock.call.page_test.RunNavigateSteps(
 198                   p, mock_shared_state.current_tab),
 199                 mock.call.page_test.DidNavigateToPage(
 200                   p, mock_shared_state.current_tab)]
 201     self.assertEquals(mock_shared_state.mock_calls, expected)
 202
 203 class PageExecutionTest(unittest.TestCase):
 204   def setupTest(self, num_test_failures=0, manager_mock=None):
 205     finder_options = fakes.CreateBrowserFinderOptions()
 206     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
 207     finder_options.output_formats = ['none']
 208     finder_options.suppress_gtest_report = True
 209     finder_options.output_dir = None
 210     finder_options.upload_bucket = 'public'
 211     finder_options.upload_results = False
 212     testclass = FakeTest
 213     parser = finder_options.CreateParser()
 214     benchmark.AddCommandLineArgs(parser)
 215     testclass.AddCommandLineArgs(parser)
 216     options, dummy_args = parser.parse_args([])
 217     benchmark.ProcessCommandLineArgs(parser, options)
 218     testclass.ProcessCommandLineArgs(parser, options)
 219     test = testclass(times_to_fail_test=num_test_failures,
 220                      manager_mock=manager_mock)
 221     return test, finder_options
 222
 223   # Test page.Run() method is called by telemetry framework before
 224   # ValidateAndMeasurePageInner.
 225   def testPageRunMethodIsCalledBeforeValidateAndMeasurePage(self):
 226     manager = mock.Mock()
 227     test, finder_options = self.setupTest(manager_mock=manager)
 228     page = FakePage(test, 'page1')
 229     page.Run = manager.Run
 230     test.AddFakePage(page)
 231     self.assertEqual(test.Run(finder_options), 0,
 232                      'Test should run with no errors')
 233     expected = [mock.call.Run(mock.ANY),
 234                 mock.call.validator.ValidateAndMeasurePageInner(
 235                   page, mock.ANY, mock.ANY)]
 236     self.assertEquals(manager.mock_calls, expected)
 237
 238   def testPassingPage(self):
 239     manager = mock.Mock()
 240     test, finder_options = self.setupTest(manager_mock=manager)
 241     page = FakePage(test, 'page1', manager_mock=manager.page)
 242     test.AddFakePage(page)
 243     self.assertEqual(test.Run(finder_options), 0,
 244                      'Test should run with no errors')
 245     expected = [mock.call.validator.WillNavigateToPageInner(
 246                   page, mock.ANY),
 247                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 248                 mock.call.validator.DidNavigateToPageInner(
 249                   page, mock.ANY),
 250                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 251                 mock.call.validator.ValidateAndMeasurePageInner(
 252                   page, mock.ANY, mock.ANY)]
 253     self.assertTrue(manager.mock_calls == expected)
 254
 255   def testFailingPage(self):
 256     test, finder_options = self.setupTest()
 257     page = FailingPage(test, 'page1')
 258     test.AddFakePage(page)
 259     self.assertNotEqual(test.Run(finder_options), 0, 'Test should fail')
 260
 261   def testExpectedFailure(self):
 262     test, finder_options = self.setupTest()
 263     page = FailingPage(test, 'page1')
 264     test.AddFakePage(page)
 265     test.GetExpectations().Fail('page1')
 266     self.assertEqual(test.Run(finder_options), 0,
 267                      'Test should run with no errors')
 268     self.assertFalse(page.RunPageInteractionsInner.called)
 269     self.assertFalse(test.validator.ValidateAndMeasurePageInner.called)
 270
 271   def testPageSetRepeatOfPageWhichFailsOnce(self):
 272     test, finder_options = self.setupTest()
 273     finder_options.pageset_repeat = 2
 274     page = PageWhichFailsNTimes(test, 'page1', 1)
 275     test.AddFakePage(page)
 276     test.GetExpectations().Fail('page1')
 277     self.assertEqual(test.Run(finder_options), 0,
 278                      'Test should run with no errors')
 279     # This will be called the second time through the page set, when
 280     # the page doesn't fail.
 281     self.assertTrue(page.RunPageInteractionsInner.called)
 282
 283   def testSkipAtPageBaseLevel(self):
 284     test, finder_options = self.setupTest()
 285     page = FailingPage(test, 'page1')
 286     test.AddFakePage(page)
 287     test.GetExpectations().Skip('page1')
 288     self.assertEqual(test.Run(finder_options), 0,
 289                      'Test should run with no errors')
 290     self.assertFalse(test.validator.WillNavigateToPageInner.called)
 291     self.assertFalse(page.RunNavigateStepsInner.called)
 292     self.assertFalse(test.validator.DidNavigateToPageInner.called)
 293     self.assertFalse(page.RunPageInteractionsInner.called)
 294     self.assertFalse(test.validator.ValidateAndMeasurePageInner.called)
 295
 296   def testSkipAtPageLevel(self):
 297     test, finder_options = self.setupTest()
 298     page = FakePage(test, 'page1')
 299     page.RunNavigateSteps = mock.Mock()
 300     page.RunPageInteractions = mock.Mock()
 301     test.validator.WillNavigateToPage = mock.Mock()
 302     test.validator.DidNavigateToPage = mock.Mock()
 303     test.validator.ValidateAndMeasurePage = mock.Mock()
 304     test.AddFakePage(page)
 305     test.GetExpectations().Skip('page1')
 306     self.assertEqual(test.Run(finder_options), 0,
 307                      'Test should run with no errors')
 308     self.assertFalse(test.validator.WillNavigateToPage.called)
 309     self.assertFalse(page.RunNavigateSteps.called)
 310     self.assertFalse(test.validator.DidNavigateToPage.called)
 311     self.assertFalse(page.RunPageInteractions.called)
 312     self.assertFalse(test.validator.ValidateAndMeasurePage.called)
 313
 314   def testPassAfterExpectedFailure(self):
 315     manager = mock.Mock()
 316     test, finder_options = self.setupTest(manager_mock=manager)
 317     page1 = FailingPage(test, 'page1', manager_mock=manager.page1)
 318     test.AddFakePage(page1)
 319     test.GetExpectations().Fail('page1')
 320     page2 = FakePage(test, 'page2', manager_mock=manager.page2)
 321     test.AddFakePage(page2)
 322     self.assertEqual(test.Run(finder_options), 0,
 323                      'Test should run with no errors')
 324     expected = [mock.call.validator.WillNavigateToPageInner(
 325                   page1, mock.ANY),
 326                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 327                 mock.call.validator.WillNavigateToPageInner(
 328                   page2, mock.ANY),
 329                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 330                 mock.call.validator.DidNavigateToPageInner(
 331                   page2, mock.ANY),
 332                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 333                 mock.call.validator.ValidateAndMeasurePageInner(
 334                   page2, mock.ANY, mock.ANY)]
 335     self.assertTrue(manager.mock_calls == expected)
 336
 337   def testExpectedDevtoolsTargetCrash(self):
 338     manager = mock.Mock()
 339     test, finder_options = self.setupTest(manager_mock=manager)
 340     page = CrashingPage(test, 'page1', manager_mock=manager.page)
 341     test.AddFakePage(page)
 342     test.GetExpectations().Fail('page1')
 343     self.assertEqual(test.Run(finder_options), 0,
 344                      'Test should run with no errors')
 345     expected = [mock.call.validator.WillNavigateToPageInner(
 346                   page, mock.ANY),
 347                 mock.call.page.RunNavigateStepsInner(mock.ANY)]
 348     self.assertTrue(manager.mock_calls == expected)
 349
 350   def testFlakyPage(self):
 351     manager = mock.Mock()
 352     test, finder_options = self.setupTest(manager_mock=manager)
 353     page = PageWhichFailsNTimes(test, 'page1', 1, manager_mock=manager.page)
 354     test.AddFakePage(page)
 355     test.GetExpectations().Flaky('page1')
 356     self.assertEqual(test.Run(finder_options), 0,
 357                      'Test should run with no errors')
 358     expected = [mock.call.validator.WillNavigateToPageInner(
 359                   page, mock.ANY),
 360                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 361                 mock.call.validator.WillNavigateToPageInner(
 362                   page, mock.ANY),
 363                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 364                 mock.call.validator.DidNavigateToPageInner(
 365                   page, mock.ANY),
 366                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 367                 mock.call.validator.ValidateAndMeasurePageInner(
 368                   page, mock.ANY, mock.ANY)]
 369     self.assertTrue(manager.mock_calls == expected)
 370
 371   def testFlakyTest(self):
 372     manager = mock.Mock()
 373     test, finder_options = self.setupTest(num_test_failures=1,
 374                                           manager_mock=manager)
 375     page = FakePage(test, 'page1', manager_mock=manager.page)
 376     test.AddFakePage(page)
 377     test.GetExpectations().Flaky('page1')
 378     self.assertEqual(test.Run(finder_options), 0,
 379                      'Test should run with no errors')
 380     expected = [mock.call.validator.WillNavigateToPageInner(
 381                   page, mock.ANY),
 382                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 383                 mock.call.validator.DidNavigateToPageInner(
 384                   page, mock.ANY),
 385                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 386                 mock.call.validator.ValidateAndMeasurePageInner(
 387                   page, mock.ANY, mock.ANY),
 388                 mock.call.validator.WillNavigateToPageInner(
 389                   page, mock.ANY),
 390                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 391                 mock.call.validator.DidNavigateToPageInner(
 392                   page, mock.ANY),
 393                 mock.call.page.RunPageInteractionsInner(mock.ANY),
 394                 mock.call.validator.ValidateAndMeasurePageInner(
 395                   page, mock.ANY, mock.ANY)]
 396     self.assertTrue(manager.mock_calls == expected)
 397
 398   def testFlakyPageExceedingNumRetries(self):
 399     manager = mock.Mock()
 400     test, finder_options = self.setupTest(manager_mock=manager)
 401     page = PageWhichFailsNTimes(test, 'page1', 2, manager_mock=manager.page)
 402     test.AddFakePage(page)
 403     test.GetExpectations().Flaky('page1', max_num_retries=1)
 404     self.assertNotEqual(test.Run(finder_options), 0,
 405                      'Test should fail')
 406     expected = [mock.call.validator.WillNavigateToPageInner(
 407                   page, mock.ANY),
 408                 mock.call.page.RunNavigateStepsInner(mock.ANY),
 409                 mock.call.validator.WillNavigateToPageInner(
 410                   page, mock.ANY),
 411                 mock.call.page.RunNavigateStepsInner(mock.ANY)]
 412     self.assertTrue(manager.mock_calls == expected)
 413
 414   def testFlakyPageThenPassingPage(self):
 415     manager = mock.Mock()
 416     test, finder_options = self.setupTest(manager_mock=manager)
 417     page1 = PageWhichFailsNTimes(test, 'page1', 1, manager_mock=manager.page1)
 418     test.AddFakePage(page1)
 419     page2 = FakePage(test, 'page2', manager_mock=manager.page2)
 420     test.AddFakePage(page2)
 421     test.GetExpectations().Flaky('page1', max_num_retries=1)
 422     self.assertEqual(test.Run(finder_options), 0,
 423                      'Test should run with no errors')
 424     expected = [mock.call.validator.WillNavigateToPageInner(
 425                   page1, mock.ANY),
 426                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 427                 mock.call.validator.WillNavigateToPageInner(
 428                   page1, mock.ANY),
 429                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 430                 mock.call.validator.DidNavigateToPageInner(
 431                   page1, mock.ANY),
 432                 mock.call.page1.RunPageInteractionsInner(mock.ANY),
 433                 mock.call.validator.ValidateAndMeasurePageInner(
 434                   page1, mock.ANY, mock.ANY),
 435                 mock.call.validator.WillNavigateToPageInner(
 436                   page2, mock.ANY),
 437                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 438                 mock.call.validator.DidNavigateToPageInner(
 439                   page2, mock.ANY),
 440                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 441                 mock.call.validator.ValidateAndMeasurePageInner(
 442                   page2, mock.ANY, mock.ANY)]
 443     self.assertTrue(manager.mock_calls == expected)
 444
 445   def testPassingPageThenFlakyPage(self):
 446     manager = mock.Mock()
 447     test, finder_options = self.setupTest(manager_mock=manager)
 448     page1 = FakePage(test, 'page1', manager_mock=manager.page1)
 449     test.AddFakePage(page1)
 450     page2 = PageWhichFailsNTimes(test, 'page2', 1, manager_mock=manager.page2)
 451     test.AddFakePage(page2)
 452     test.GetExpectations().Flaky('page2', max_num_retries=1)
 453     self.assertEqual(test.Run(finder_options), 0,
 454                      'Test should run with no errors')
 455     expected = [mock.call.validator.WillNavigateToPageInner(
 456                   page1, mock.ANY),
 457                 mock.call.page1.RunNavigateStepsInner(mock.ANY),
 458                 mock.call.validator.DidNavigateToPageInner(
 459                   page1, mock.ANY),
 460                 mock.call.page1.RunPageInteractionsInner(mock.ANY),
 461                 mock.call.validator.ValidateAndMeasurePageInner(
 462                   page1, mock.ANY, mock.ANY),
 463                 mock.call.validator.WillNavigateToPageInner(
 464                   page2, mock.ANY),
 465                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 466                 mock.call.validator.WillNavigateToPageInner(
 467                   page2, mock.ANY),
 468                 mock.call.page2.RunNavigateStepsInner(mock.ANY),
 469                 mock.call.validator.DidNavigateToPageInner(
 470                   page2, mock.ANY),
 471                 mock.call.page2.RunPageInteractionsInner(mock.ANY),
 472                 mock.call.validator.ValidateAndMeasurePageInner(
 473                   page2, mock.ANY, mock.ANY)]
 474     self.assertTrue(manager.mock_calls == expected)