content/test/gpu/gpu_tests/gpu_test_base.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6
   7 from telemetry import benchmark as benchmark_module
   8 from telemetry.core import exceptions
   9 from telemetry.page import page as page_module
  10 from telemetry.page import page_test
  11 from telemetry.page import shared_page_state
  12 from telemetry.value import skip
  13
  14 import exception_formatter
  15 import gpu_test_expectations
  16
  17 """Base classes for all GPU tests in this directory. Implements
  18 support for per-page test expectations."""
  19
  20 def _PageOperationWrapper(page, tab, expectations,
  21                           show_expected_failure, inner_func,
  22                           results=None):
  23   """Wraps an operation to be done against the page. If an error has
  24   occurred in an earlier step, skips this entirely."""
  25   if page.HadError():
  26     return
  27   expectation = 'pass'
  28   if expectations:
  29     expectation = expectations.GetExpectationForPage(tab.browser, page)
  30   if expectation == 'skip':
  31     raise Exception(
  32       'Skip expectations should have been handled at a higher level')
  33   try:
  34     inner_func()
  35   except Exception:
  36     page.SetHadError()
  37     if expectation == 'pass':
  38       raise
  39     elif expectation == 'fail':
  40       msg = 'Expected exception while running %s' % page.display_name
  41       exception_formatter.PrintFormattedException(msg=msg)
  42     elif expectation == 'flaky':
  43       if not page.GetSuppressFlakyFailures():
  44         raise
  45     else:
  46       logging.warning(
  47           'Unknown expectation %s while handling exception for %s' %
  48           (expectation, page.display_name))
  49       raise
  50   else:
  51     if show_expected_failure and expectation == 'fail':
  52       logging.warning(
  53           '%s was expected to fail, but passed.\n', page.display_name)
  54
  55
  56 class TestBase(benchmark_module.Benchmark):
  57   def __init__(self, max_failures=None):
  58     super(TestBase, self).__init__(max_failures=max_failures)
  59     self._cached_expectations = None
  60
  61   def GetExpectations(self):
  62     """Returns the expectations that apply to this test."""
  63     if not self._cached_expectations:
  64       self._cached_expectations = self._CreateExpectations()
  65     if not isinstance(self._cached_expectations,
  66                       gpu_test_expectations.GpuTestExpectations):
  67       raise Exception('gpu_test_base requires use of GpuTestExpectations')
  68     return self._cached_expectations
  69
  70   def _CreateExpectations(self):
  71     # By default, creates an empty GpuTestExpectations object. Override
  72     # this in subclasses to set up test-specific expectations. Must
  73     # return an instance of GpuTestExpectations or a subclass.
  74     #
  75     # Do not call this directly. Call GetExpectations where necessary.
  76     return gpu_test_expectations.GpuTestExpectations()
  77
  78
  79 class ValidatorBase(page_test.PageTest):
  80   def __init__(self,
  81                needs_browser_restart_after_each_page=False,
  82                clear_cache_before_each_run=False):
  83     super(ValidatorBase, self).__init__(
  84       needs_browser_restart_after_each_page=\
  85         needs_browser_restart_after_each_page,
  86       clear_cache_before_each_run=clear_cache_before_each_run)
  87
  88   def WillNavigateToPage(self, page, tab):
  89     """Does operations before the page is navigated. Do not override this
  90     method. Override WillNavigateToPageInner, below."""
  91     _PageOperationWrapper(
  92       page, tab, page.GetExpectations(), False,
  93       lambda: self.WillNavigateToPageInner(page, tab))
  94
  95   def DidNavigateToPage(self, page, tab):
  96     """Does operations right after the page is navigated and after all
  97     waiting for completion has occurred. Do not override this method.
  98     Override DidNavigateToPageInner, below."""
  99     _PageOperationWrapper(
 100       page, tab, page.GetExpectations(), False,
 101       lambda: self.DidNavigateToPageInner(page, tab))
 102
 103   def ValidateAndMeasurePage(self, page, tab, results):
 104     """Validates and measures the page, taking into account test
 105     expectations. Do not override this method. Override
 106     ValidateAndMeasurePageInner, below."""
 107
 108     # First figure out whether this test is marked flaky in the test
 109     # expectations.
 110     num_retries = page.GetExpectations().GetFlakyRetriesForPage(
 111       tab.browser, page)
 112
 113     try:
 114       _PageOperationWrapper(
 115         page, tab, page.GetExpectations(), True,
 116         lambda: self.ValidateAndMeasurePageInner(page, tab, results),
 117         results=results)
 118     except Exception:
 119       # If we're going to re-execute a flaky test in the finally
 120       # clause, then we have to squelch this exception.
 121       if num_retries == 0:
 122         raise
 123     finally:
 124       if page.HadError() and num_retries:
 125         # Emulate the logic that's in shared_page_state to re-run the
 126         # test up to |num_retries| times. It would be ideal if it
 127         # could be reused (by calling SharedPageState.RunStory), but
 128         # unfortunately it's not possible to fetch the active
 129         # SharedPageState on demand.
 130         for ii in xrange(0, num_retries):
 131           print 'FLAKY TEST FAILURE, retrying: ' + page.display_name
 132           page.ClearHadError()
 133           page.SetSuppressFlakyFailures(False)
 134           try:
 135             self.WillNavigateToPage(page, tab)
 136             page.RunNavigateSteps(page.cached_action_runner)
 137             self.DidNavigateToPage(page, tab)
 138             page.RunPageInteractions(page.cached_action_runner)
 139             # Must not re-enter ourselves!
 140             _PageOperationWrapper(
 141               page, tab, page.GetExpectations(), True,
 142               lambda: self.ValidateAndMeasurePageInner(page, tab, results),
 143               results=results)
 144           except Exception:
 145             # Squelch any exceptions from any but the last retry.
 146             if ii == num_retries - 1:
 147               raise
 148           finally:
 149             # If the retry succeeded, stop.
 150             if not page.HadError():
 151               break
 152             # Otherwise, clear the error state and retry.
 153             page.ClearHadError()
 154       # Clear the error state of the page at this point so that if
 155       # --page-repeat or --pageset-repeat are used, the subsequent
 156       # iterations don't turn into no-ops.
 157       page.ClearHadError()
 158
 159   def WillNavigateToPageInner(self, page, tab):
 160     pass
 161
 162   def DidNavigateToPageInner(self, page, tab):
 163     pass
 164
 165   def ValidateAndMeasurePageInner(self, page, tab, results):
 166     pass
 167
 168
 169 # NOTE: if you change this logic you must change the logic in
 170 # FakeGpuSharedPageState (gpu_test_base_unittest.py) as well.
 171 def _CanRunOnBrowser(browser_info, page):
 172   expectations = page.GetExpectations()
 173   return expectations.GetExpectationForPage(
 174     browser_info.browser, page) != 'skip'
 175
 176
 177 class GpuSharedPageState(shared_page_state.SharedPageState):
 178   def CanRunOnBrowser(self, browser_info, page):
 179     return _CanRunOnBrowser(browser_info, page)
 180
 181
 182 # TODO(kbr): re-evaluate the need for this SharedPageState
 183 # subclass. It's only used by the WebGL conformance suite.
 184 class DesktopGpuSharedPageState(
 185     shared_page_state.SharedDesktopPageState):
 186   def CanRunOnBrowser(self, browser_info, page):
 187     return _CanRunOnBrowser(browser_info, page)
 188
 189
 190 class PageBase(page_module.Page):
 191   # The convention is that pages subclassing this class must be
 192   # configured with the test expectations.
 193   def __init__(self, url, page_set=None, base_dir=None, name='',
 194                shared_page_state_class=GpuSharedPageState,
 195                make_javascript_deterministic=True,
 196                expectations=None):
 197     super(PageBase, self).__init__(
 198       url=url, page_set=page_set, base_dir=base_dir, name=name,
 199       shared_page_state_class=shared_page_state_class,
 200       make_javascript_deterministic=make_javascript_deterministic)
 201     # Disable automatic garbage collection to reduce the test's cycle time.
 202     self._collect_garbage_before_run = False
 203
 204     # TODO(kbr): this is fragile -- if someone changes the
 205     # shared_page_state_class to something that doesn't handle skip
 206     # expectations, then they'll hit the exception in
 207     # _PageOperationWrapper, above. Need to rethink.
 208     self._expectations = expectations
 209     self._had_error = False
 210     self._cached_action_runner = None
 211     self._suppress_flaky_failures = True
 212
 213   def GetExpectations(self):
 214     return self._expectations
 215
 216   def HadError(self):
 217     return self._had_error
 218
 219   def SetHadError(self):
 220     self._had_error = True
 221
 222   def ClearHadError(self):
 223     self._had_error = False
 224
 225   def SetSuppressFlakyFailures(self, value):
 226     self._suppress_flaky_failures = value
 227
 228   def GetSuppressFlakyFailures(self):
 229     return self._suppress_flaky_failures
 230
 231   @property
 232   def cached_action_runner(self):
 233     return self._cached_action_runner
 234
 235   def RunNavigateSteps(self, action_runner):
 236     """Runs navigation steps, taking into account test expectations.
 237     Do not override this method. Override RunNavigateStepsInner, below."""
 238     self._cached_action_runner = action_runner
 239     def Functor():
 240       self.RunDefaultNavigateSteps(action_runner)
 241       self.RunNavigateStepsInner(action_runner)
 242     _PageOperationWrapper(self, action_runner.tab, self.GetExpectations(),
 243                           False, Functor)
 244
 245   def RunDefaultNavigateSteps(self, action_runner):
 246     """Runs the default set of navigation steps inherited from page.Page."""
 247     super(PageBase, self).RunNavigateSteps(action_runner)
 248
 249   def RunNavigateStepsInner(self, action_runner):
 250     pass
 251
 252   def RunPageInteractions(self, action_runner):
 253     """Runs page interactions, taking into account test expectations. Do not
 254     override this method. Override RunPageInteractionsInner, below."""
 255     def Functor():
 256       super(PageBase, self).RunPageInteractions(action_runner)
 257       self.RunPageInteractionsInner(action_runner)
 258     _PageOperationWrapper(self, action_runner.tab, self.GetExpectations(),
 259                           False, Functor)
 260
 261   def RunPageInteractionsInner(self, action_runner):
 262     pass