content/test/gpu/gpu_tests/test_expectations_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import unittest
   5
   6 import test_expectations
   7
   8 from telemetry.page import page as page_module
   9 from telemetry.story import story_set
  10
  11 class StubPlatform(object):
  12   def __init__(self, os_name, os_version_name=None):
  13     self.os_name = os_name
  14     self.os_version_name = os_version_name
  15
  16   def GetOSName(self):
  17     return self.os_name
  18
  19   def GetOSVersionName(self):
  20     return self.os_version_name
  21
  22
  23 class StubBrowser(object):
  24   def __init__(self, platform, browser_type=None):
  25     self.platform = platform
  26     self.browser_type = browser_type
  27
  28   @property
  29   def supports_system_info(self):
  30     return False
  31
  32   def GetSystemInfo(self):
  33     return self.system_info
  34
  35
  36 class SampleExpectationSubclass(test_expectations.Expectation):
  37   def __init__(self, expectation, pattern, conditions=None, bug=None):
  38     self.valid_condition_matched = False
  39     self.valid_condition_unmatched = False
  40     super(SampleExpectationSubclass, self).__init__(
  41       expectation, pattern, conditions=conditions, bug=bug)
  42
  43   def ParseCondition(self, condition):
  44     if condition == 'valid_condition_matched':
  45       self.valid_condition_matched = True
  46     elif condition == 'valid_condition_unmatched':
  47       self.valid_condition_unmatched = True
  48     else:
  49       super(SampleExpectationSubclass, self).ParseCondition(condition)
  50
  51
  52 class SampleTestExpectations(test_expectations.TestExpectations):
  53   def CreateExpectation(self, expectation, url_pattern, conditions=None,
  54                         bug=None):
  55     return SampleExpectationSubclass(expectation, url_pattern,
  56                                      conditions=conditions, bug=bug)
  57
  58   def SetExpectations(self):
  59     self.Fail('page1.html', ['win', 'mac'], bug=123)
  60     self.Fail('page2.html', ['vista'], bug=123)
  61     self.Fail('page3.html', bug=123)
  62     self.Fail('page4.*', bug=123)
  63     self.Fail('http://test.com/page5.html', bug=123)
  64     self.Fail('Pages.page_6')
  65     self.Fail('page7.html', ['mountainlion'])
  66     self.Fail('page8.html', ['mavericks'])
  67     self.Fail('page9.html', ['yosemite'])
  68     # Test browser conditions.
  69     self.Fail('page10.html', ['android', 'android-webview-shell'], bug=456)
  70     # Test user defined conditions.
  71     self.Fail('page11.html', ['win', 'valid_condition_matched'])
  72     self.Fail('page12.html', ['win', 'valid_condition_unmatched'])
  73     # Test file:// scheme.
  74     self.Fail('conformance/attribs/page13.html')
  75     # Test file:// scheme on Windows.
  76     self.Fail('conformance/attribs/page14.html', ['win'])
  77     # Explicitly matched paths have precedence over wildcards.
  78     self.Fail('conformance/glsl/*')
  79     self.Skip('conformance/glsl/page15.html')
  80
  81   def ExpectationAppliesToPage(self, expectation, browser, page):
  82     if not super(SampleTestExpectations,
  83         self).ExpectationAppliesToPage(expectation, browser, page):
  84       return False
  85     # These tests can probably be expressed more tersely, but that
  86     # would reduce readability.
  87     if (not expectation.valid_condition_matched and
  88         not expectation.valid_condition_unmatched):
  89       return True
  90     return expectation.valid_condition_matched
  91
  92 class TestExpectationsTest(unittest.TestCase):
  93   def setUp(self):
  94     self.expectations = SampleTestExpectations()
  95
  96   def assertExpectationEquals(self, expected, page, platform=StubPlatform(''),
  97                               browser_type=None):
  98     self.expectations.ClearExpectationsCacheForTesting()
  99     result = self.expectations.GetExpectationForPage(
 100       StubBrowser(platform, browser_type=browser_type), page)
 101     self.assertEquals(expected, result)
 102
 103   # Pages with no expectations should always return 'pass'
 104   def testNoExpectations(self):
 105     ps = story_set.StorySet()
 106     page = page_module.Page('http://test.com/page0.html', ps)
 107     self.assertExpectationEquals('pass', page, StubPlatform('win'))
 108
 109   # Pages with expectations for an OS should only return them when running on
 110   # that OS
 111   def testOSExpectations(self):
 112     ps = story_set.StorySet()
 113     page = page_module.Page('http://test.com/page1.html', ps)
 114     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 115     self.assertExpectationEquals('fail', page, StubPlatform('mac'))
 116     self.assertExpectationEquals('pass', page, StubPlatform('linux'))
 117
 118   # Pages with expectations for an OS version should only return them when
 119   # running on that OS version
 120   def testOSVersionExpectations(self):
 121     ps = story_set.StorySet()
 122     page = page_module.Page('http://test.com/page2.html', ps)
 123     self.assertExpectationEquals('fail', page, StubPlatform('win', 'vista'))
 124     self.assertExpectationEquals('pass', page, StubPlatform('win', 'win7'))
 125
 126   # Pages with non-conditional expectations should always return that
 127   # expectation regardless of OS or OS version
 128   def testConditionlessExpectations(self):
 129     ps = story_set.StorySet()
 130     page = page_module.Page('http://test.com/page3.html', ps)
 131     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 132     self.assertExpectationEquals('fail', page, StubPlatform('mac', 'lion'))
 133     self.assertExpectationEquals('fail', page, StubPlatform('linux'))
 134
 135   # Expectations with wildcard characters should return for matching patterns
 136   def testWildcardExpectations(self):
 137     ps = story_set.StorySet()
 138     page = page_module.Page('http://test.com/page4.html', ps)
 139     page_js = page_module.Page('http://test.com/page4.html', ps)
 140     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 141     self.assertExpectationEquals('fail', page_js, StubPlatform('win'))
 142
 143   # Expectations with absolute paths should match the entire path
 144   def testAbsoluteExpectations(self):
 145     ps = story_set.StorySet()
 146     page = page_module.Page('http://test.com/page5.html', ps)
 147     page_org = page_module.Page('http://test.org/page5.html', ps)
 148     page_https = page_module.Page('https://test.com/page5.html', ps)
 149     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 150     self.assertExpectationEquals('pass', page_org, StubPlatform('win'))
 151     self.assertExpectationEquals('pass', page_https, StubPlatform('win'))
 152
 153   # Expectations can be set against page names as well as urls
 154   def testPageNameExpectations(self):
 155     ps = story_set.StorySet()
 156     page = page_module.Page('http://test.com/page6.html', ps,
 157                             name='Pages.page_6')
 158     self.assertExpectationEquals('fail', page)
 159
 160   # Verify version-specific Mac expectations.
 161   def testMacVersionExpectations(self):
 162     ps = story_set.StorySet()
 163     page = page_module.Page('http://test.com/page7.html', ps)
 164     self.assertExpectationEquals('fail', page,
 165                                  StubPlatform('mac', 'mountainlion'))
 166     self.assertExpectationEquals('pass', page,
 167                                  StubPlatform('mac', 'mavericks'))
 168     self.assertExpectationEquals('pass', page,
 169                                  StubPlatform('mac', 'yosemite'))
 170     ps = story_set.StorySet()
 171     page = page_module.Page('http://test.com/page8.html', ps)
 172     self.assertExpectationEquals('pass', page,
 173                                  StubPlatform('mac', 'mountainlion'))
 174     self.assertExpectationEquals('fail', page,
 175                                  StubPlatform('mac', 'mavericks'))
 176     self.assertExpectationEquals('pass', page,
 177                                  StubPlatform('mac', 'yosemite'))
 178     ps = story_set.StorySet()
 179     page = page_module.Page('http://test.com/page9.html', ps)
 180     self.assertExpectationEquals('pass', page,
 181                                  StubPlatform('mac', 'mountainlion'))
 182     self.assertExpectationEquals('pass', page,
 183                                  StubPlatform('mac', 'mavericks'))
 184     self.assertExpectationEquals('fail', page,
 185                                  StubPlatform('mac', 'yosemite'))
 186
 187   # Test browser type conditions.
 188   def testBrowserTypeConditions(self):
 189     ps = story_set.StorySet()
 190     page = page_module.Page('http://test.com/page10.html', ps)
 191     self.assertExpectationEquals('pass', page, StubPlatform('android'),
 192                                  browser_type='android-content-shell')
 193     self.assertExpectationEquals('fail', page, StubPlatform('android'),
 194                                  browser_type='android-webview-shell')
 195
 196   # Pages with user-defined conditions.
 197   def testUserDefinedConditions(self):
 198     ps = story_set.StorySet()
 199     page = page_module.Page('http://test.com/page11.html', ps)
 200     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 201     page = page_module.Page('http://test.com/page12.html', ps)
 202     self.assertExpectationEquals('pass', page, StubPlatform('win'))
 203
 204   # The file:// scheme is treated specially by Telemetry; it's
 205   # translated into an HTTP request against localhost. Expectations
 206   # against it must continue to work.
 207   def testFileScheme(self):
 208     ps = story_set.StorySet()
 209     page = page_module.Page('file://conformance/attribs/page13.html', ps)
 210     self.assertExpectationEquals('fail', page)
 211
 212   # Telemetry uses backslashes in its file:// URLs on Windows.
 213   def testFileSchemeOnWindows(self):
 214     ps = story_set.StorySet()
 215     page = page_module.Page('file://conformance\\attribs\\page14.html', ps)
 216     self.assertExpectationEquals('pass', page)
 217     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 218
 219   def testExplicitPathsHavePrecedenceOverWildcards(self):
 220     ps = story_set.StorySet()
 221     page = page_module.Page('http://test.com/conformance/glsl/page00.html', ps)
 222     self.assertExpectationEquals('fail', page)
 223     page = page_module.Page('http://test.com/conformance/glsl/page15.html', ps)
 224     self.assertExpectationEquals('skip', page)