content/test/gpu/gpu_tests/gpu_test_base.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6
   7 from telemetry import benchmark as benchmark_module
   8 from telemetry.core import exceptions
   9 from telemetry.page import page as page_module
  10 from telemetry.page import page_test
  11 from telemetry.page import shared_page_state
  12 from telemetry.value import skip
  13
  14 import exception_formatter
  15 import gpu_test_expectations
  16
  17 """Base classes for all GPU tests in this directory. Implements
  18 support for per-page test expectations."""
  19
  20 def _PageOperationWrapper(page, tab, expectations,
  21                           show_expected_failure, inner_func,
  22                           results=None):
  23   """Wraps an operation to be done against the page. If an error has
  24   occurred in an earlier step, skips this entirely."""
  25   if page.HadError():
  26     return
  27   expectation = 'pass'
  28   if expectations:
  29     expectation = expectations.GetExpectationForPage(tab.browser, page)
  30   if expectation == 'skip':
  31     raise Exception(
  32       'Skip expectations should have been handled at a higher level')
  33   try:
  34     inner_func()
  35   except Exception:
  36     page.SetHadError()
  37     if expectation == 'pass':
  38       raise
  39     elif expectation == 'fail':
  40       msg = 'Expected exception while running %s' % page.display_name
  41       exception_formatter.PrintFormattedException(msg=msg)
  42     elif expectation == 'flaky':
  43       # Retries are handled in RunStoryWithRetries, below.
  44       raise
  45     else:
  46       logging.warning(
  47           'Unknown expectation %s while handling exception for %s' %
  48           (expectation, page.display_name))
  49       raise
  50   else:
  51     if show_expected_failure and expectation == 'fail':
  52       logging.warning(
  53           '%s was expected to fail, but passed.\n', page.display_name)
  54
  55
  56 class TestBase(benchmark_module.Benchmark):
  57   def __init__(self, max_failures=None):
  58     super(TestBase, self).__init__(max_failures=max_failures)
  59     self._cached_expectations = None
  60
  61   def GetExpectations(self):
  62     """Returns the expectations that apply to this test."""
  63     if not self._cached_expectations:
  64       self._cached_expectations = self._CreateExpectations()
  65     if not isinstance(self._cached_expectations,
  66                       gpu_test_expectations.GpuTestExpectations):
  67       raise Exception('gpu_test_base requires use of GpuTestExpectations')
  68     return self._cached_expectations
  69
  70   def _CreateExpectations(self):
  71     # By default, creates an empty GpuTestExpectations object. Override
  72     # this in subclasses to set up test-specific expectations. Must
  73     # return an instance of GpuTestExpectations or a subclass.
  74     #
  75     # Do not call this directly. Call GetExpectations where necessary.
  76     return gpu_test_expectations.GpuTestExpectations()
  77
  78
  79 class ValidatorBase(page_test.PageTest):
  80   def __init__(self,
  81                needs_browser_restart_after_each_page=False,
  82                clear_cache_before_each_run=False):
  83     super(ValidatorBase, self).__init__(
  84       needs_browser_restart_after_each_page=\
  85         needs_browser_restart_after_each_page,
  86       clear_cache_before_each_run=clear_cache_before_each_run)
  87
  88   def WillNavigateToPage(self, page, tab):
  89     """Does operations before the page is navigated. Do not override this
  90     method. Override WillNavigateToPageInner, below."""
  91     _PageOperationWrapper(
  92       page, tab, page.GetExpectations(), False,
  93       lambda: self.WillNavigateToPageInner(page, tab))
  94
  95   def DidNavigateToPage(self, page, tab):
  96     """Does operations right after the page is navigated and after all
  97     waiting for completion has occurred. Do not override this method.
  98     Override DidNavigateToPageInner, below."""
  99     _PageOperationWrapper(
 100       page, tab, page.GetExpectations(), False,
 101       lambda: self.DidNavigateToPageInner(page, tab))
 102
 103   def ValidateAndMeasurePage(self, page, tab, results):
 104     """Validates and measures the page, taking into account test
 105     expectations. Do not override this method. Override
 106     ValidateAndMeasurePageInner, below."""
 107     _PageOperationWrapper(
 108       page, tab, page.GetExpectations(), True,
 109       lambda: self.ValidateAndMeasurePageInner(page, tab, results),
 110       results=results)
 111
 112   def WillNavigateToPageInner(self, page, tab):
 113     pass
 114
 115   def DidNavigateToPageInner(self, page, tab):
 116     pass
 117
 118   def ValidateAndMeasurePageInner(self, page, tab, results):
 119     pass
 120
 121
 122 # NOTE: if you change this logic you must change the logic in
 123 # FakeGpuSharedPageState (gpu_test_base_unittest.py) as well.
 124 def _CanRunOnBrowser(browser_info, page):
 125   expectations = page.GetExpectations()
 126   return expectations.GetExpectationForPage(
 127     browser_info.browser, page) != 'skip'
 128
 129 def RunStoryWithRetries(cls, shared_page_state, results):
 130   page = shared_page_state.current_page
 131   try:
 132     super(cls, shared_page_state).RunStory(results)
 133   except Exception:
 134     tab = shared_page_state.current_tab
 135     num_retries = page.GetExpectations().GetFlakyRetriesForPage(
 136       tab.browser, page)
 137     if page.HadError() and num_retries:
 138       # Re-run the test up to |num_retries| times.
 139       page.ClearHadError()
 140       for ii in xrange(0, num_retries):
 141         print 'FLAKY TEST FAILURE, retrying: ' + page.display_name
 142         try:
 143           super(cls, shared_page_state).RunStory(results)
 144           break
 145         except Exception:
 146           # Squelch any exceptions from any but the last retry.
 147           if ii == num_retries - 1:
 148             raise
 149         finally:
 150           # Clear the error state in case we have to retry.
 151           page.ClearHadError()
 152     else:
 153       # Re-raise the exception.
 154       raise
 155   finally:
 156     # Clear the error state of the page at this point so that if
 157     # --page-repeat or --pageset-repeat are used, the subsequent
 158     # iterations don't turn into no-ops.
 159     page.ClearHadError()
 160
 161 class GpuSharedPageState(shared_page_state.SharedPageState):
 162   def CanRunOnBrowser(self, browser_info, page):
 163     return _CanRunOnBrowser(browser_info, page)
 164
 165   # NOTE: if you change this logic you must change the logic in
 166   # FakeGpuSharedPageState (gpu_test_base_unittest.py) as well.
 167   def RunStory(self, results):
 168     RunStoryWithRetries(GpuSharedPageState, self, results)
 169
 170
 171 # TODO(kbr): re-evaluate the need for this SharedPageState
 172 # subclass. It's only used by the WebGL conformance suite.
 173 class DesktopGpuSharedPageState(
 174     shared_page_state.SharedDesktopPageState):
 175   def CanRunOnBrowser(self, browser_info, page):
 176     return _CanRunOnBrowser(browser_info, page)
 177
 178   # Note: if you change this logic you must change the logic in
 179   # FakeGpuSharedPageState (gpu_test_base_unittest.py) as well.
 180   def RunStory(self, results):
 181     RunStoryWithRetries(DesktopGpuSharedPageState, self, results)
 182
 183
 184 class PageBase(page_module.Page):
 185   # The convention is that pages subclassing this class must be
 186   # configured with the test expectations.
 187   def __init__(self, url, page_set=None, base_dir=None, name='',
 188                shared_page_state_class=GpuSharedPageState,
 189                make_javascript_deterministic=True,
 190                expectations=None):
 191     super(PageBase, self).__init__(
 192       url=url, page_set=page_set, base_dir=base_dir, name=name,
 193       shared_page_state_class=shared_page_state_class,
 194       make_javascript_deterministic=make_javascript_deterministic)
 195     # Disable automatic garbage collection to reduce the test's cycle time.
 196     self._collect_garbage_before_run = False
 197
 198     # TODO(kbr): this is fragile -- if someone changes the
 199     # shared_page_state_class to something that doesn't handle skip
 200     # expectations, then they'll hit the exception in
 201     # _PageOperationWrapper, above. Need to rethink.
 202     self._expectations = expectations
 203     self._had_error = False
 204     self._cached_action_runner = None
 205
 206   def GetExpectations(self):
 207     return self._expectations
 208
 209   def HadError(self):
 210     return self._had_error
 211
 212   def SetHadError(self):
 213     self._had_error = True
 214
 215   def ClearHadError(self):
 216     self._had_error = False
 217
 218   @property
 219   def cached_action_runner(self):
 220     return self._cached_action_runner
 221
 222   def RunNavigateSteps(self, action_runner):
 223     """Runs navigation steps, taking into account test expectations.
 224     Do not override this method. Override RunNavigateStepsInner, below."""
 225     self._cached_action_runner = action_runner
 226     def Functor():
 227       self.RunDefaultNavigateSteps(action_runner)
 228       self.RunNavigateStepsInner(action_runner)
 229     _PageOperationWrapper(self, action_runner.tab, self.GetExpectations(),
 230                           False, Functor)
 231
 232   def RunDefaultNavigateSteps(self, action_runner):
 233     """Runs the default set of navigation steps inherited from page.Page."""
 234     super(PageBase, self).RunNavigateSteps(action_runner)
 235
 236   def RunNavigateStepsInner(self, action_runner):
 237     pass
 238
 239   def RunPageInteractions(self, action_runner):
 240     """Runs page interactions, taking into account test expectations. Do not
 241     override this method. Override RunPageInteractionsInner, below."""
 242     def Functor():
 243       super(PageBase, self).RunPageInteractions(action_runner)
 244       self.RunPageInteractionsInner(action_runner)
 245     _PageOperationWrapper(self, action_runner.tab, self.GetExpectations(),
 246                           False, Functor)
 247
 248   def RunPageInteractionsInner(self, action_runner):
 249     pass