content/test/gpu/gpu_tests/test_expectations.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import fnmatch
   6 import urlparse
   7
   8 # Valid expectation conditions are:
   9 #
  10 # Operating systems:
  11 #     win, xp, vista, win7, win8, win10, mac, leopard, snowleopard,
  12 #     lion, mountainlion, mavericks, yosemite, linux, chromeos,
  13 #     android
  14 #
  15 # Browser types:
  16 #     android-webview-shell, android-content-shell, debug, release
  17 #
  18 # Sample usage in SetExpectations in subclasses:
  19 #   self.Fail('gl-enable-vertex-attrib.html',
  20 #       ['mac', 'release'], bug=123)
  21
  22 OS_CONDITIONS = ['win', 'xp', 'vista', 'win7', 'win8', 'win10',
  23                  'mac', 'leopard', 'snowleopard', 'lion', 'mountainlion',
  24                  'mavericks', 'yosemite', 'linux', 'chromeos', 'android']
  25
  26 BROWSER_TYPE_CONDITIONS = [
  27     'android-webview-shell', 'android-content-shell', 'debug', 'release' ]
  28
  29 class Expectation(object):
  30   """Represents a single test expectation for a page.
  31
  32   Supports conditions based on operating system (e.g., win, mac) and
  33   browser type (e.g. 'debug', 'release').
  34
  35   Subclass this class and call super.__init__ last in your constructor
  36   in order to add new user-defined conditions. The conditions are
  37   parsed at the end of this class's constructor, so be careful not to
  38   overwrite the results of the constructor call!
  39   """
  40
  41   def __init__(self, expectation, pattern, conditions=None, bug=None):
  42     self.expectation = expectation.lower()
  43     self.pattern = pattern
  44     self.bug = bug
  45
  46     self.os_conditions = []
  47     self.browser_conditions = []
  48
  49     if conditions:
  50       for c in conditions:
  51         self.ParseCondition(c)
  52
  53   def ParseCondition(self, condition):
  54     """Parses a single test expectation condition.
  55
  56     Can be overridden to handle new types of conditions. Call the
  57     superclass's implementation of ParseCondition at the end of your
  58     subclass if you don't handle the condition. The base
  59     implementation will raise an exception if the condition is
  60     unsupported.
  61
  62     Valid expectation conditions are:
  63
  64     Operating systems:
  65       win, xp, vista, win7, mac, leopard, snowleopard, lion,
  66       mountainlion, mavericks, yosemite, linux, chromeos, android
  67
  68     Browser types:
  69       android-webview-shell, android-content-shell, debug, release
  70
  71     Sample usage in SetExpectations in subclasses:
  72       self.Fail('gl-enable-vertex-attrib.html',
  73          ['mac', 'release'], bug=123)
  74     """
  75     cl = condition.lower()
  76     if cl in OS_CONDITIONS:
  77       self.os_conditions.append(cl)
  78     elif cl in BROWSER_TYPE_CONDITIONS:
  79       self.browser_conditions.append(condition)
  80     else:
  81       raise ValueError('Unknown expectation condition: "%s"' % cl)
  82
  83
  84 class TestExpectations(object):
  85   """A class which defines the expectations for a page set test execution"""
  86
  87   def __init__(self):
  88     self._expectations = []
  89     self._skip_matching_names = False
  90     self._built_expectation_cache = True
  91     self._ClearExpectationsCache()
  92     self.SetExpectations()
  93
  94   def SetExpectations(self):
  95     """Called on creation. Override to set up custom expectations."""
  96     pass
  97
  98   def Fail(self, pattern, conditions=None, bug=None):
  99     self._Expect('fail', pattern, conditions, bug)
 100
 101   def Skip(self, pattern, conditions=None, bug=None):
 102     self._Expect('skip', pattern, conditions, bug)
 103
 104   def _Expect(self, expectation, pattern, conditions=None, bug=None):
 105     self._AddExpectation(self.CreateExpectation(expectation, pattern,
 106                                                 conditions, bug))
 107
 108   def _AddExpectation(self, expectation):
 109     '''Call this to add an expectation to the set.
 110
 111     For use only by this class and subclasses. Do not call this directly.'''
 112     self._expectations.append(expectation)
 113     self._ClearExpectationsCache()
 114
 115
 116   def CreateExpectation(self, expectation, pattern, conditions=None,
 117                         bug=None):
 118     return Expectation(expectation, pattern, conditions, bug)
 119
 120   def _ClearExpectationsCache(self):
 121     if self._built_expectation_cache:
 122       # Only those expectations which contain no wildcard characters
 123       # (those which the fnmatch module would expand).
 124       self._expectations_by_pattern = {}
 125       # The remaining expectations which require expansion.
 126       self._expectations_with_wildcards = []
 127       self._built_expectation_cache = False
 128
 129   def ClearExpectationsCacheForTesting(self):
 130     '''For use only by unit tests.'''
 131     self._ClearExpectationsCache()
 132
 133   def _HasWildcardCharacters(self, input_string):
 134     # Could make this more precise.
 135     return '*' in input_string or '+' in input_string
 136
 137   def _BuildExpectationsCache(self, browser, page):
 138     # Turn off name matching while building the cache.
 139     self._skip_matching_names = True
 140     for e in self._expectations:
 141       if self.ExpectationAppliesToPage(e, browser, page):
 142         if (self._HasWildcardCharacters(e.pattern)):
 143           self._expectations_with_wildcards.append(e)
 144         else:
 145           self._expectations_by_pattern[e.pattern] = e
 146     self._built_expectation_cache = True
 147     self._skip_matching_names = False
 148
 149   def _GetNormalizedURL(self, url, browser):
 150     # Telemetry uses backslashes in its file:// URLs on Windows,
 151     # breaking matching of test expectations.
 152     if not browser.platform.GetOSName() == 'win':
 153       return url
 154     return url.replace('\\', '/')
 155
 156   def _GetURLPath(self, url):
 157     components = urlparse.urlsplit(url)
 158     # For compatibility, the file:// scheme must be treated specially.
 159     # The top-level directory shows up in the netloc portion of the URL.
 160     if components[0] == 'file':
 161       url_path = components[1] + components[2]
 162     else:
 163       url_path = components[2]
 164     # Chop any leading slash since the expectations used by this class
 165     # assume that.
 166     if (url_path and url_path[0] == '/'):
 167       url_path = url_path[1:]
 168     return url_path
 169
 170   def _GetExpectationObjectForPage(self, browser, page):
 171     if not self._built_expectation_cache:
 172       self._BuildExpectationsCache(browser, page)
 173     # First attempt to look up by the page's URL or name.
 174     e = None
 175     # Relative URL (common case).
 176     url = self._GetNormalizedURL(page.url, browser)
 177     url_path = self._GetURLPath(url)
 178     if url_path:
 179       e = self._expectations_by_pattern.get(url_path)
 180     if e:
 181       return e
 182     e = self._expectations_by_pattern.get(url)
 183     if e:
 184       return e
 185     if page.name:
 186       e = self._expectations_by_pattern.get(page.name)
 187     if e:
 188       return e
 189     # Fall back to scanning through the expectations containing
 190     # wildcards.
 191     for e in self._expectations_with_wildcards:
 192       if self.ExpectationAppliesToPage(e, browser, page):
 193         return e
 194     return None
 195
 196   def GetExpectationForPage(self, browser, page):
 197     '''Fetches the expectation that applies to the given page.
 198
 199     The implementation of this function performs significant caching
 200     based on the browser's parameters, which are expected to remain
 201     unchanged from call to call. If this is not true, the method
 202     ClearExpectationsCacheForTesting is available to clear the cache;
 203     but file a bug if this is needed for any reason but testing.
 204     '''
 205     e = self._GetExpectationObjectForPage(browser, page)
 206     if e:
 207       return e.expectation
 208     return 'pass'
 209
 210   def ExpectationAppliesToPage(self, expectation, browser, page):
 211     """Defines whether the given expectation applies to the given page.
 212
 213     Override this in subclasses to add more conditions. Call the
 214     superclass's implementation first, and return false if it returns
 215     false. Subclasses must not consult the page's name or URL; that is
 216     the responsibility of the base class.
 217
 218     Args:
 219       expectation: an instance of a subclass of Expectation, created
 220           by a call to CreateExpectation.
 221       browser: the currently running browser.
 222       page: the page to be run.
 223     """
 224     # While building the expectations cache we need to match
 225     # everything except the page's name or URL.
 226     if not self._skip_matching_names:
 227       # Relative URL.
 228       if not fnmatch.fnmatch(self._GetURLPath(page.url),
 229                              expectation.pattern):
 230         # Absolute URL.
 231         if not fnmatch.fnmatch(page.url,
 232                                expectation.pattern):
 233           # Name.
 234           if not (page.name and fnmatch.fnmatch(page.name,
 235                                                 expectation.pattern)):
 236             return False
 237
 238     platform = browser.platform
 239     os_matches = (not expectation.os_conditions or
 240         platform.GetOSName() in expectation.os_conditions or
 241         platform.GetOSVersionName() in expectation.os_conditions)
 242
 243     browser_matches = (
 244       (not expectation.browser_conditions) or
 245       browser.browser_type in expectation.browser_conditions)
 246
 247     return os_matches and browser_matches