content/test/gpu/gpu_tests/test_expectations_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import unittest
   5
   6 import test_expectations
   7
   8 from telemetry import story
   9 from telemetry.internal.platform import system_info
  10 from telemetry.page import page as page_module
  11
  12 VENDOR_NVIDIA = 0x10DE
  13 VENDOR_AMD = 0x1002
  14 VENDOR_INTEL = 0x8086
  15
  16 VENDOR_STRING_IMAGINATION = 'Imagination Technologies'
  17 DEVICE_STRING_SGX = 'PowerVR SGX 554'
  18
  19 class StubPlatform(object):
  20   def __init__(self, os_name, os_version_name=None):
  21     self.os_name = os_name
  22     self.os_version_name = os_version_name
  23
  24   def GetOSName(self):
  25     return self.os_name
  26
  27   def GetOSVersionName(self):
  28     return self.os_version_name
  29
  30 class StubBrowser(object):
  31   def __init__(self, platform, gpu, device, vendor_string, device_string):
  32     self.platform = platform
  33     self.system_info = system_info.SystemInfo.FromDict({
  34       'model_name': '',
  35       'gpu': {
  36         'devices': [
  37           {'vendor_id': gpu, 'device_id': device,
  38            'vendor_string': vendor_string, 'device_string': device_string},
  39         ]
  40       }
  41     })
  42
  43   @property
  44   def supports_system_info(self):
  45     return False if not self.system_info else True
  46
  47   def GetSystemInfo(self):
  48     return self.system_info
  49
  50 class SampleTestExpectations(test_expectations.TestExpectations):
  51   def SetExpectations(self):
  52     self.Fail('page1.html', ['win', 'mac'], bug=123)
  53     self.Fail('page2.html', ['vista'], bug=123)
  54     self.Fail('page3.html', bug=123)
  55     self.Fail('page4.*', bug=123)
  56     self.Fail('http://test.com/page5.html', bug=123)
  57     self.Fail('page6.html', ['nvidia', 'intel'], bug=123)
  58     self.Fail('page7.html', [('nvidia', 0x1001), ('nvidia', 0x1002)], bug=123)
  59     self.Fail('page8.html', ['win', 'intel', ('amd', 0x1001)], bug=123)
  60     self.Fail('page9.html', ['imagination'])
  61     self.Fail('page10.html', [('imagination', 'PowerVR SGX 554')])
  62     self.Fail('Pages.page_11')
  63     self.Fail('page12.html', ['mountainlion'])
  64     self.Fail('page13.html', ['mavericks'])
  65     self.Fail('page14.html', ['yosemite'])
  66     self.Fail('page15.html', ['amd', 'valid_condition_matched'])
  67     self.Fail('page16.html', ['amd', 'valid_condition_unmatched'])
  68
  69   def IsValidUserDefinedCondition(self, condition):
  70     return condition in ('valid_condition_matched', 'valid_condition_unmatched')
  71
  72   def ModifiersApply(self, shared_page_state, expectation):
  73     if not super(SampleTestExpectations,
  74         self).ModifiersApply(shared_page_state, expectation):
  75       return False
  76     return ((not expectation.user_defined_conditions) or
  77         'valid_condition_matched' in expectation.user_defined_conditions)
  78
  79 class TestExpectationsTest(unittest.TestCase):
  80   def setUp(self):
  81     self.expectations = SampleTestExpectations()
  82
  83   def assertExpectationEquals(self, expected, page, platform='', gpu=0,
  84       device=0, vendor_string='', device_string=''):
  85     result = self.expectations.GetExpectationForPage(StubBrowser(
  86       platform, gpu, device, vendor_string, device_string), page)
  87     self.assertEquals(expected, result)
  88
  89   # Pages with no expectations should always return 'pass'
  90   def testNoExpectations(self):
  91     story_set = story.StorySet()
  92     page = page_module.Page('http://test.com/page0.html', story_set)
  93     self.assertExpectationEquals('pass', page, StubPlatform('win'))
  94
  95   # Pages with expectations for an OS should only return them when running on
  96   # that OS
  97   def testOSExpectations(self):
  98     story_set = story.StorySet()
  99     page = page_module.Page('http://test.com/page1.html', story_set)
 100     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 101     self.assertExpectationEquals('fail', page, StubPlatform('mac'))
 102     self.assertExpectationEquals('pass', page, StubPlatform('linux'))
 103
 104   # Pages with expectations for an OS version should only return them when
 105   # running on that OS version
 106   def testOSVersionExpectations(self):
 107     story_set = story.StorySet()
 108     page = page_module.Page('http://test.com/page2.html', story_set)
 109     self.assertExpectationEquals('fail', page, StubPlatform('win', 'vista'))
 110     self.assertExpectationEquals('pass', page, StubPlatform('win', 'win7'))
 111
 112   # Pages with non-conditional expectations should always return that
 113   # expectation regardless of OS or OS version
 114   def testConditionlessExpectations(self):
 115     story_set = story.StorySet()
 116     page = page_module.Page('http://test.com/page3.html', story_set)
 117     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 118     self.assertExpectationEquals('fail', page, StubPlatform('mac', 'lion'))
 119     self.assertExpectationEquals('fail', page, StubPlatform('linux'))
 120
 121   # Expectations with wildcard characters should return for matching patterns
 122   def testWildcardExpectations(self):
 123     story_set = story.StorySet()
 124     page = page_module.Page('http://test.com/page4.html', story_set)
 125     page_js = page_module.Page('http://test.com/page4.html', story_set)
 126     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 127     self.assertExpectationEquals('fail', page_js, StubPlatform('win'))
 128
 129   # Expectations with absolute paths should match the entire path
 130   def testAbsoluteExpectations(self):
 131     story_set = story.StorySet()
 132     page = page_module.Page('http://test.com/page5.html', story_set)
 133     page_org = page_module.Page('http://test.org/page5.html', story_set)
 134     page_https = page_module.Page('https://test.com/page5.html', story_set)
 135     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 136     self.assertExpectationEquals('pass', page_org, StubPlatform('win'))
 137     self.assertExpectationEquals('pass', page_https, StubPlatform('win'))
 138
 139   # Pages with expectations for a GPU should only return them when running with
 140   # that GPU
 141   def testGpuExpectations(self):
 142     story_set = story.StorySet()
 143     page = page_module.Page('http://test.com/page6.html', story_set)
 144     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA)
 145     self.assertExpectationEquals('fail', page, gpu=VENDOR_INTEL)
 146     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD)
 147
 148   # Pages with expectations for a GPU should only return them when running with
 149   # that GPU
 150   def testGpuDeviceIdExpectations(self):
 151     story_set = story.StorySet()
 152     page = page_module.Page('http://test.com/page7.html', story_set)
 153     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1001)
 154     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1002)
 155     self.assertExpectationEquals('pass', page, gpu=VENDOR_NVIDIA, device=0x1003)
 156     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD, device=0x1001)
 157
 158   # Pages with multiple expectations should only return them when all criteria
 159   # is met
 160   def testMultipleExpectations(self):
 161     story_set = story.StorySet()
 162     page = page_module.Page('http://test.com/page8.html', story_set)
 163     self.assertExpectationEquals('fail', page,
 164         StubPlatform('win'), VENDOR_AMD, 0x1001)
 165     self.assertExpectationEquals('fail', page,
 166         StubPlatform('win'), VENDOR_INTEL, 0x1002)
 167     self.assertExpectationEquals('pass', page,
 168         StubPlatform('win'), VENDOR_NVIDIA, 0x1001)
 169     self.assertExpectationEquals('pass', page,
 170         StubPlatform('mac'), VENDOR_AMD, 0x1001)
 171     self.assertExpectationEquals('pass', page,
 172         StubPlatform('win'), VENDOR_AMD, 0x1002)
 173
 174   # Pages with expectations based on GPU vendor string.
 175   def testGpuVendorStringExpectations(self):
 176     story_set = story.StorySet()
 177     page = page_module.Page('http://test.com/page9.html', story_set)
 178     self.assertExpectationEquals('fail', page,
 179                                  vendor_string=VENDOR_STRING_IMAGINATION,
 180                                  device_string=DEVICE_STRING_SGX)
 181     self.assertExpectationEquals('fail', page,
 182                                  vendor_string=VENDOR_STRING_IMAGINATION,
 183                                  device_string='Triangle Monster 3000')
 184     self.assertExpectationEquals('pass', page,
 185                                  vendor_string='Acme',
 186                                  device_string=DEVICE_STRING_SGX)
 187
 188   # Pages with expectations based on GPU vendor and renderer string pairs.
 189   def testGpuDeviceStringExpectations(self):
 190     story_set = story.StorySet()
 191     page = page_module.Page('http://test.com/page10.html', story_set)
 192     self.assertExpectationEquals('fail', page,
 193                                  vendor_string=VENDOR_STRING_IMAGINATION,
 194                                  device_string=DEVICE_STRING_SGX)
 195     self.assertExpectationEquals('pass', page,
 196                                  vendor_string=VENDOR_STRING_IMAGINATION,
 197                                  device_string='Triangle Monster 3000')
 198     self.assertExpectationEquals('pass', page,
 199                                  vendor_string='Acme',
 200                                  device_string=DEVICE_STRING_SGX)
 201
 202   # Pages with user-defined expectations.
 203   def testUserDefinedExpectations(self):
 204     story_set = story.StorySet()
 205     page = page_module.Page('http://test.com/page15.html', story_set)
 206     self.assertExpectationEquals('fail', page, gpu=VENDOR_AMD)
 207     page = page_module.Page('http://test.com/page16.html', story_set)
 208     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD)
 209
 210   # Expectations can be set against page names as well as urls
 211   def testPageNameExpectations(self):
 212     story_set = story.StorySet()
 213     page = page_module.Page('http://test.com/page11.html', story_set,
 214                             name='Pages.page_11')
 215     self.assertExpectationEquals('fail', page)
 216
 217   # Verify version-specific Mac expectations.
 218   def testMacVersionExpectations(self):
 219     story_set = story.StorySet()
 220     page = page_module.Page('http://test.com/page12.html', story_set)
 221     self.assertExpectationEquals('fail', page,
 222                                  StubPlatform('mac', 'mountainlion'))
 223     self.assertExpectationEquals('pass', page,
 224                                  StubPlatform('mac', 'mavericks'))
 225     self.assertExpectationEquals('pass', page,
 226                                  StubPlatform('mac', 'yosemite'))
 227     story_set = story.StorySet()
 228     page = page_module.Page('http://test.com/page13.html', story_set)
 229     self.assertExpectationEquals('pass', page,
 230                                  StubPlatform('mac', 'mountainlion'))
 231     self.assertExpectationEquals('fail', page,
 232                                  StubPlatform('mac', 'mavericks'))
 233     self.assertExpectationEquals('pass', page,
 234                                  StubPlatform('mac', 'yosemite'))
 235     story_set = story.StorySet()
 236     page = page_module.Page('http://test.com/page14.html', story_set)
 237     self.assertExpectationEquals('pass', page,
 238                                  StubPlatform('mac', 'mountainlion'))
 239     self.assertExpectationEquals('pass', page,
 240                                  StubPlatform('mac', 'mavericks'))
 241     self.assertExpectationEquals('fail', page,
 242                                  StubPlatform('mac', 'yosemite'))