content/test/gpu/gpu_tests/gpu_test_expectations_unittest.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import unittest
   5
   6 from telemetry.internal.platform import system_info
   7 from telemetry.page import page as page_module
   8 from telemetry.story import story_set
   9
  10 import gpu_test_expectations
  11
  12 VENDOR_NVIDIA = 0x10DE
  13 VENDOR_AMD = 0x1002
  14 VENDOR_INTEL = 0x8086
  15
  16 VENDOR_STRING_IMAGINATION = 'Imagination Technologies'
  17 DEVICE_STRING_SGX = 'PowerVR SGX 554'
  18
  19 class StubPlatform(object):
  20   def __init__(self, os_name, os_version_name=None):
  21     self.os_name = os_name
  22     self.os_version_name = os_version_name
  23
  24   def GetOSName(self):
  25     return self.os_name
  26
  27   def GetOSVersionName(self):
  28     return self.os_version_name
  29
  30 class StubBrowser(object):
  31   def __init__(self, platform, gpu, device, vendor_string, device_string,
  32                browser_type=None, gl_renderer=None):
  33     self.platform = platform
  34     self.browser_type = browser_type
  35     sys_info = {
  36       'model_name': '',
  37       'gpu': {
  38         'devices': [
  39           {'vendor_id': gpu, 'device_id': device,
  40            'vendor_string': vendor_string, 'device_string': device_string},
  41         ]
  42       }
  43     }
  44     if gl_renderer:
  45       sys_info['gpu']['aux_attributes'] = {
  46         'gl_renderer': gl_renderer
  47       }
  48     self.system_info = system_info.SystemInfo.FromDict(sys_info)
  49
  50   @property
  51   def supports_system_info(self):
  52     return False if not self.system_info else True
  53
  54   def GetSystemInfo(self):
  55     return self.system_info
  56
  57 class SampleTestExpectations(gpu_test_expectations.GpuTestExpectations):
  58   def SetExpectations(self):
  59     # Test GPU conditions.
  60     self.Fail('test1.html', ['nvidia', 'intel'], bug=123)
  61     self.Fail('test2.html', [('nvidia', 0x1001), ('nvidia', 0x1002)], bug=123)
  62     self.Fail('test3.html', ['win', 'intel', ('amd', 0x1001)], bug=123)
  63     self.Fail('test4.html', ['imagination'])
  64     self.Fail('test5.html', [('imagination', 'PowerVR SGX 554')])
  65     # Test ANGLE conditions.
  66     self.Fail('test6.html', ['win', 'd3d9'], bug=345)
  67     # Test flaky expectations.
  68     self.Flaky('test7.html', bug=123, max_num_retries=5)
  69     self.Flaky('test8.html', ['win'], bug=123, max_num_retries=6)
  70     self.Flaky('wildcardtest*.html', ['win'], bug=123, max_num_retries=7)
  71
  72 class GpuTestExpectationsTest(unittest.TestCase):
  73   def setUp(self):
  74     self.expectations = SampleTestExpectations()
  75
  76   def assertExpectationEquals(self, expected, page, platform=StubPlatform(''),
  77                               gpu=0, device=0, vendor_string='',
  78                               device_string='', browser_type=None,
  79                               gl_renderer=None):
  80     self.expectations.ClearExpectationsCacheForTesting()
  81     result = self.expectations.GetExpectationForPage(StubBrowser(
  82       platform, gpu, device, vendor_string, device_string,
  83       browser_type=browser_type, gl_renderer=gl_renderer), page)
  84     self.assertEquals(expected, result)
  85
  86   def getRetriesForPage(self, page, platform=StubPlatform(''), gpu=0,
  87       device=0, vendor_string='', device_string=''):
  88     self.expectations.ClearExpectationsCacheForTesting()
  89     return self.expectations.GetFlakyRetriesForPage(StubBrowser(
  90       platform, gpu, device, vendor_string, device_string), page)
  91
  92   # Pages with expectations for a GPU should only return them when running with
  93   # that GPU
  94   def testGpuExpectations(self):
  95     ps = story_set.StorySet()
  96     page = page_module.Page('http://test.com/test1.html', ps)
  97     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA)
  98     self.assertExpectationEquals('fail', page, gpu=VENDOR_INTEL)
  99     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD)
 100
 101   # Pages with expectations for a GPU should only return them when running with
 102   # that GPU
 103   def testGpuDeviceIdExpectations(self):
 104     ps = story_set.StorySet()
 105     page = page_module.Page('http://test.com/test2.html', ps)
 106     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1001)
 107     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1002)
 108     self.assertExpectationEquals('pass', page, gpu=VENDOR_NVIDIA, device=0x1003)
 109     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD, device=0x1001)
 110
 111   # Pages with multiple expectations should only return them when all criteria
 112   # are met
 113   def testMultipleExpectations(self):
 114     ps = story_set.StorySet()
 115     page = page_module.Page('http://test.com/test3.html', ps)
 116     self.assertExpectationEquals('fail', page,
 117         StubPlatform('win'), VENDOR_AMD, 0x1001)
 118     self.assertExpectationEquals('fail', page,
 119         StubPlatform('win'), VENDOR_INTEL, 0x1002)
 120     self.assertExpectationEquals('pass', page,
 121         StubPlatform('win'), VENDOR_NVIDIA, 0x1001)
 122     self.assertExpectationEquals('pass', page,
 123         StubPlatform('mac'), VENDOR_AMD, 0x1001)
 124     self.assertExpectationEquals('pass', page,
 125         StubPlatform('win'), VENDOR_AMD, 0x1002)
 126
 127   # Pages with expectations based on GPU vendor string.
 128   def testGpuVendorStringExpectations(self):
 129     ps = story_set.StorySet()
 130     page = page_module.Page('http://test.com/test4.html', ps)
 131     self.assertExpectationEquals('fail', page,
 132                                  vendor_string=VENDOR_STRING_IMAGINATION,
 133                                  device_string=DEVICE_STRING_SGX)
 134     self.assertExpectationEquals('fail', page,
 135                                  vendor_string=VENDOR_STRING_IMAGINATION,
 136                                  device_string='Triangle Monster 3000')
 137     self.assertExpectationEquals('pass', page,
 138                                  vendor_string='Acme',
 139                                  device_string=DEVICE_STRING_SGX)
 140
 141   # Pages with expectations based on GPU vendor and renderer string pairs.
 142   def testGpuDeviceStringExpectations(self):
 143     ps = story_set.StorySet()
 144     page = page_module.Page('http://test.com/test5.html', ps)
 145     self.assertExpectationEquals('fail', page,
 146                                  vendor_string=VENDOR_STRING_IMAGINATION,
 147                                  device_string=DEVICE_STRING_SGX)
 148     self.assertExpectationEquals('pass', page,
 149                                  vendor_string=VENDOR_STRING_IMAGINATION,
 150                                  device_string='Triangle Monster 3000')
 151     self.assertExpectationEquals('pass', page,
 152                                  vendor_string='Acme',
 153                                  device_string=DEVICE_STRING_SGX)
 154
 155   # Test ANGLE conditions.
 156   def testANGLEConditions(self):
 157     ps = story_set.StorySet()
 158     page = page_module.Page('http://test.com/test6.html', ps)
 159     self.assertExpectationEquals('pass', page, StubPlatform('win'),
 160                                  gl_renderer='Direct3D11')
 161     self.assertExpectationEquals('fail', page, StubPlatform('win'),
 162                                  gl_renderer='Direct3D9')
 163
 164   # Ensure retry mechanism is working.
 165   def testFlakyExpectation(self):
 166     ps = story_set.StorySet()
 167     page = page_module.Page('http://test.com/test7.html', ps)
 168     self.assertExpectationEquals('flaky', page)
 169     self.assertEquals(5, self.getRetriesForPage(page))
 170
 171   # Ensure the filtering from the TestExpectations superclass still works.
 172   def testFlakyPerPlatformExpectation(self):
 173     ps = story_set.StorySet()
 174     page1 = page_module.Page('http://test.com/test8.html', ps)
 175     self.assertExpectationEquals('flaky', page1, StubPlatform('win'))
 176     self.assertEquals(6, self.getRetriesForPage(page1, StubPlatform('win')))
 177     self.assertExpectationEquals('pass', page1, StubPlatform('mac'))
 178     self.assertEquals(0, self.getRetriesForPage(page1, StubPlatform('mac')))
 179     page2 = page_module.Page('http://test.com/wildcardtest1.html', ps)
 180     self.assertExpectationEquals('flaky', page2, StubPlatform('win'))
 181     self.assertEquals(7, self.getRetriesForPage(page2, StubPlatform('win')))
 182     self.assertExpectationEquals('pass', page2, StubPlatform('mac'))
 183     self.assertEquals(0, self.getRetriesForPage(page2, StubPlatform('mac')))