content/test/gpu/gpu_tests/test_expectations.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import fnmatch
   6 import urlparse
   7
   8 # Valid expectation conditions are:
   9 #
  10 # Operating systems:
  11 #     win, xp, vista, win7, mac, leopard, snowleopard, lion, mountainlion,
  12 #     mavericks, yosemite, linux, chromeos, android
  13 #
  14 # Browser types:
  15 #     android-webview-shell, android-content-shell, debug, release
  16 #
  17 # Sample usage in SetExpectations in subclasses:
  18 #   self.Fail('gl-enable-vertex-attrib.html',
  19 #       ['mac', 'release'], bug=123)
  20
  21 OS_CONDITIONS = ['win', 'xp', 'vista', 'win7',
  22                  'mac', 'leopard', 'snowleopard', 'lion', 'mountainlion',
  23                  'mavericks', 'yosemite', 'linux', 'chromeos', 'android']
  24
  25 BROWSER_TYPE_CONDITIONS = [
  26     'android-webview-shell', 'android-content-shell', 'debug', 'release' ]
  27
  28 class Expectation(object):
  29   """Represents a single test expectation for a page.
  30
  31   Supports conditions based on operating system (e.g., win, mac) and
  32   browser type (e.g. 'debug', 'release').
  33
  34   Subclass this class and call super.__init__ last in your constructor
  35   in order to add new user-defined conditions. The conditions are
  36   parsed at the end of this class's constructor, so be careful not to
  37   overwrite the results of the constructor call!
  38   """
  39
  40   def __init__(self, expectation, pattern, conditions=None, bug=None):
  41     self.expectation = expectation.lower()
  42     self.pattern = pattern
  43     self.bug = bug
  44
  45     self.os_conditions = []
  46     self.browser_conditions = []
  47
  48     if conditions:
  49       for c in conditions:
  50         self.ParseCondition(c)
  51
  52   def ParseCondition(self, condition):
  53     """Parses a single test expectation condition.
  54
  55     Can be overridden to handle new types of conditions. Call the
  56     superclass's implementation of ParseCondition at the end of your
  57     subclass if you don't handle the condition. The base
  58     implementation will raise an exception if the condition is
  59     unsupported.
  60
  61     Valid expectation conditions are:
  62
  63     Operating systems:
  64       win, xp, vista, win7, mac, leopard, snowleopard, lion,
  65       mountainlion, mavericks, yosemite, linux, chromeos, android
  66
  67     Browser types:
  68       android-webview-shell, android-content-shell, debug, release
  69
  70     Sample usage in SetExpectations in subclasses:
  71       self.Fail('gl-enable-vertex-attrib.html',
  72          ['mac', 'release'], bug=123)
  73     """
  74     cl = condition.lower()
  75     if cl in OS_CONDITIONS:
  76       self.os_conditions.append(cl)
  77     elif cl in BROWSER_TYPE_CONDITIONS:
  78       self.browser_conditions.append(condition)
  79     else:
  80       raise ValueError('Unknown expectation condition: "%s"' % cl)
  81
  82
  83 class TestExpectations(object):
  84   """A class which defines the expectations for a page set test execution"""
  85
  86   def __init__(self):
  87     self._expectations = []
  88     self._skip_matching_names = False
  89     self._built_expectation_cache = True
  90     self._ClearExpectationsCache()
  91     self.SetExpectations()
  92
  93   def SetExpectations(self):
  94     """Called on creation. Override to set up custom expectations."""
  95     pass
  96
  97   def Fail(self, pattern, conditions=None, bug=None):
  98     self._Expect('fail', pattern, conditions, bug)
  99
 100   def Skip(self, pattern, conditions=None, bug=None):
 101     self._Expect('skip', pattern, conditions, bug)
 102
 103   def _Expect(self, expectation, pattern, conditions=None, bug=None):
 104     self._AddExpectation(self.CreateExpectation(expectation, pattern,
 105                                                 conditions, bug))
 106
 107   def _AddExpectation(self, expectation):
 108     '''Call this to add an expectation to the set.
 109
 110     For use only by this class and subclasses. Do not call this directly.'''
 111     self._expectations.append(expectation)
 112     self._ClearExpectationsCache()
 113
 114
 115   def CreateExpectation(self, expectation, pattern, conditions=None,
 116                         bug=None):
 117     return Expectation(expectation, pattern, conditions, bug)
 118
 119   def _ClearExpectationsCache(self):
 120     if self._built_expectation_cache:
 121       # Only those expectations which contain no wildcard characters
 122       # (those which the fnmatch module would expand).
 123       self._expectations_by_pattern = {}
 124       # The remaining expectations which require expansion.
 125       self._expectations_with_wildcards = []
 126       self._built_expectation_cache = False
 127
 128   def ClearExpectationsCacheForTesting(self):
 129     '''For use only by unit tests.'''
 130     self._ClearExpectationsCache()
 131
 132   def _HasWildcardCharacters(self, input_string):
 133     # Could make this more precise.
 134     return '*' in input_string or '+' in input_string
 135
 136   def _BuildExpectationsCache(self, browser, page):
 137     # Turn off name matching while building the cache.
 138     self._skip_matching_names = True
 139     for e in self._expectations:
 140       if self.ExpectationAppliesToPage(e, browser, page):
 141         if (self._HasWildcardCharacters(e.pattern)):
 142           self._expectations_with_wildcards.append(e)
 143         else:
 144           self._expectations_by_pattern[e.pattern] = e
 145     self._built_expectation_cache = True
 146     self._skip_matching_names = False
 147
 148   def _GetNormalizedURL(self, url, browser):
 149     # Telemetry uses backslashes in its file:// URLs on Windows,
 150     # breaking matching of test expectations.
 151     if not browser.platform.GetOSName() == 'win':
 152       return url
 153     return url.replace('\\', '/')
 154
 155   def _GetURLPath(self, url):
 156     components = urlparse.urlsplit(url)
 157     # For compatibility, the file:// scheme must be treated specially.
 158     # The top-level directory shows up in the netloc portion of the URL.
 159     if components[0] == 'file':
 160       url_path = components[1] + components[2]
 161     else:
 162       url_path = components[2]
 163     # Chop any leading slash since the expectations used by this class
 164     # assume that.
 165     if (url_path and url_path[0] == '/'):
 166       url_path = url_path[1:]
 167     return url_path
 168
 169   def _GetExpectationObjectForPage(self, browser, page):
 170     if not self._built_expectation_cache:
 171       self._BuildExpectationsCache(browser, page)
 172     # First attempt to look up by the page's URL or name.
 173     e = None
 174     # Relative URL (common case).
 175     url = self._GetNormalizedURL(page.url, browser)
 176     url_path = self._GetURLPath(url)
 177     if url_path:
 178       e = self._expectations_by_pattern.get(url_path)
 179     if e:
 180       return e
 181     e = self._expectations_by_pattern.get(url)
 182     if e:
 183       return e
 184     if page.name:
 185       e = self._expectations_by_pattern.get(page.name)
 186     if e:
 187       return e
 188     # Fall back to scanning through the expectations containing
 189     # wildcards.
 190     for e in self._expectations_with_wildcards:
 191       if self.ExpectationAppliesToPage(e, browser, page):
 192         return e
 193     return None
 194
 195   def GetExpectationForPage(self, browser, page):
 196     '''Fetches the expectation that applies to the given page.
 197
 198     The implementation of this function performs significant caching
 199     based on the browser's parameters, which are expected to remain
 200     unchanged from call to call. If this is not true, the method
 201     ClearExpectationsCacheForTesting is available to clear the cache;
 202     but file a bug if this is needed for any reason but testing.
 203     '''
 204     e = self._GetExpectationObjectForPage(browser, page)
 205     if e:
 206       return e.expectation
 207     return 'pass'
 208
 209   def ExpectationAppliesToPage(self, expectation, browser, page):
 210     """Defines whether the given expectation applies to the given page.
 211
 212     Override this in subclasses to add more conditions. Call the
 213     superclass's implementation first, and return false if it returns
 214     false. Subclasses must not consult the page's name or URL; that is
 215     the responsibility of the base class.
 216
 217     Args:
 218       expectation: an instance of a subclass of Expectation, created
 219           by a call to CreateExpectation.
 220       browser: the currently running browser.
 221       page: the page to be run.
 222     """
 223     # While building the expectations cache we need to match
 224     # everything except the page's name or URL.
 225     if not self._skip_matching_names:
 226       # Relative URL.
 227       if not fnmatch.fnmatch(self._GetURLPath(page.url),
 228                              expectation.pattern):
 229         # Absolute URL.
 230         if not fnmatch.fnmatch(page.url,
 231                                expectation.pattern):
 232           # Name.
 233           if not (page.name and fnmatch.fnmatch(page.name,
 234                                                 expectation.pattern)):
 235             return False
 236
 237     platform = browser.platform
 238     os_matches = (not expectation.os_conditions or
 239         platform.GetOSName() in expectation.os_conditions or
 240         platform.GetOSVersionName() in expectation.os_conditions)
 241
 242     browser_matches = (
 243       (not expectation.browser_conditions) or
 244       browser.browser_type in expectation.browser_conditions)
 245
 246     return os_matches and browser_matches