content/test/gpu/gpu_tests/webgl_conformance.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import json
   5 import optparse
   6 import os
   7 import sys
   8
   9 import webgl_conformance_expectations
  10
  11 from telemetry import benchmark as benchmark_module
  12 from telemetry.core import util
  13 from telemetry.internal.browser import browser_finder
  14 from telemetry.page import page as page_module
  15 from telemetry.page import page_test
  16 from telemetry.page import shared_page_state
  17 from telemetry.story.story_set import StorySet
  18
  19
  20 conformance_path = os.path.join(
  21     util.GetChromiumSrcDir(),
  22     'third_party', 'webgl', 'src', 'sdk', 'tests')
  23
  24 conformance_harness_script = r"""
  25   var testHarness = {};
  26   testHarness._allTestSucceeded = true;
  27   testHarness._messages = '';
  28   testHarness._failures = 0;
  29   testHarness._finished = false;
  30   testHarness._originalLog = window.console.log;
  31
  32   testHarness.log = function(msg) {
  33     testHarness._messages += msg + "\n";
  34     testHarness._originalLog.apply(window.console, [msg]);
  35   }
  36
  37   testHarness.reportResults = function(url, success, msg) {
  38     testHarness._allTestSucceeded = testHarness._allTestSucceeded && !!success;
  39     if(!success) {
  40       testHarness._failures++;
  41       if(msg) {
  42         testHarness.log(msg);
  43       }
  44     }
  45   };
  46   testHarness.notifyFinished = function(url) {
  47     testHarness._finished = true;
  48   };
  49   testHarness.navigateToPage = function(src) {
  50     var testFrame = document.getElementById("test-frame");
  51     testFrame.src = src;
  52   };
  53
  54   window.webglTestHarness = testHarness;
  55   window.parent.webglTestHarness = testHarness;
  56   window.console.log = testHarness.log;
  57   window.onerror = function(message, url, line) {
  58     testHarness.reportResults(null, false, message);
  59     testHarness.notifyFinished(null);
  60   };
  61 """
  62
  63 def _DidWebGLTestSucceed(tab):
  64   return tab.EvaluateJavaScript('webglTestHarness._allTestSucceeded')
  65
  66 def _WebGLTestMessages(tab):
  67   return tab.EvaluateJavaScript('webglTestHarness._messages')
  68
  69 def _CompareVersion(version1, version2):
  70   ver_num1 = [int(x) for x in version1.split('.')]
  71   ver_num2 = [int(x) for x in version2.split('.')]
  72   size = min(len(ver_num1), len(ver_num2))
  73   return cmp(ver_num1[0:size], ver_num2[0:size])
  74
  75 class WebglConformanceValidator(page_test.PageTest):
  76   def __init__(self):
  77     super(WebglConformanceValidator, self).__init__()
  78
  79   def ValidateAndMeasurePage(self, page, tab, results):
  80     if not _DidWebGLTestSucceed(tab):
  81       raise page_test.Failure(_WebGLTestMessages(tab))
  82
  83   def CustomizeBrowserOptions(self, options):
  84     options.AppendExtraBrowserArgs([
  85         '--disable-gesture-requirement-for-media-playback',
  86         '--disable-domain-blocking-for-3d-apis',
  87         '--disable-gpu-process-crash-limit'
  88     ])
  89     browser = browser_finder.FindBrowser(options.finder_options)
  90     if (browser.target_os.startswith('android') and
  91         browser.browser_type == 'android-webview-shell'):
  92         # TODO(kbr): this is overly broad. We'd like to do this only on
  93         # Nexus 9. It'll go away shortly anyway. crbug.com/499928
  94         #
  95         # The --ignore_egl_sync_failures is only there to work around
  96         # some strange failure on the Nexus 9 bot, not reproducible on
  97         # local hardware.
  98         options.AppendExtraBrowserArgs([
  99             '--disable-gl-extensions=GL_EXT_disjoint_timer_query',
 100             '--ignore_egl_sync_failures'
 101         ])
 102
 103
 104 class Webgl2ConformanceValidator(WebglConformanceValidator):
 105   def __init__(self):
 106     super(Webgl2ConformanceValidator, self).__init__()
 107
 108   def CustomizeBrowserOptions(self, options):
 109     options.AppendExtraBrowserArgs([
 110         '--disable-gesture-requirement-for-media-playback',
 111         '--disable-domain-blocking-for-3d-apis',
 112         '--disable-gpu-process-crash-limit',
 113         '--enable-unsafe-es3-apis'
 114     ])
 115
 116 class WebglConformancePage(page_module.Page):
 117   def __init__(self, story_set, test, expectations):
 118     super(WebglConformancePage, self).__init__(
 119       url='file://' + test, page_set=story_set, base_dir=story_set.base_dir,
 120       shared_page_state_class=shared_page_state.SharedDesktopPageState,
 121       name=('WebglConformance.%s' %
 122               test.replace('/', '_').replace('-', '_').
 123                  replace('\\', '_').rpartition('.')[0].replace('.', '_')))
 124     self.script_to_evaluate_on_commit = conformance_harness_script
 125     self._expectations = expectations
 126
 127   def RunNavigateSteps(self, action_runner):
 128     num_tries = 1 + self._expectations.GetFlakyRetriesForPage(
 129       self, action_runner.tab.browser)
 130     # This loop will run once for tests that aren't marked flaky, and
 131     # will fall through to the validator's ValidateAndMeasurePage on
 132     # the last iteration.
 133     for ii in xrange(0, num_tries):
 134       super(WebglConformancePage, self).RunNavigateSteps(action_runner)
 135       action_runner.WaitForJavaScriptCondition(
 136           'webglTestHarness._finished', timeout_in_seconds=180)
 137       if ii < num_tries - 1:
 138         if _DidWebGLTestSucceed(action_runner.tab):
 139           return
 140         else:
 141           print 'FLAKY TEST FAILURE, retrying: ' + self.display_name
 142           print 'Error messages from test run:'
 143           print _WebGLTestMessages(action_runner.tab)
 144
 145 class WebglConformance(benchmark_module.Benchmark):
 146   """Conformance with Khronos WebGL Conformance Tests"""
 147   def __init__(self):
 148     super(WebglConformance, self).__init__(max_failures=10)
 149     self._cached_expectations = None
 150
 151   @classmethod
 152   def Name(cls):
 153     return 'webgl_conformance'
 154
 155   @classmethod
 156   def AddBenchmarkCommandLineArgs(cls, group):
 157     group.add_option('--webgl-conformance-version',
 158         help='Version of the WebGL conformance tests to run.',
 159         default='1.0.4')
 160     group.add_option('--webgl2-only',
 161         help='Whether we include webgl 1 tests if version is 2.0.0 or above.',
 162         default='false')
 163
 164   def CreatePageTest(self, options):
 165     if _CompareVersion(options.webgl_conformance_version, '2.0.0') >= 0:
 166       return Webgl2ConformanceValidator()
 167     return WebglConformanceValidator()
 168
 169   def CreateStorySet(self, options):
 170     tests = self._ParseTests('00_test_list.txt',
 171         options.webgl_conformance_version,
 172         (options.webgl2_only == 'true'),
 173         None)
 174
 175     ps = StorySet(serving_dirs=[''], base_dir=conformance_path)
 176
 177     expectations = self.GetExpectations()
 178     for test in tests:
 179       ps.AddStory(WebglConformancePage(ps, test, expectations))
 180
 181     return ps
 182
 183   def GetExpectations(self):
 184     if not self._cached_expectations:
 185       self._cached_expectations = (
 186         webgl_conformance_expectations.WebGLConformanceExpectations(
 187           conformance_path))
 188     return self._cached_expectations
 189
 190   def CreateExpectations(self):
 191     return self.GetExpectations()
 192
 193   @staticmethod
 194   def _ParseTests(path, version, webgl2_only, folder_min_version):
 195     test_paths = []
 196     current_dir = os.path.dirname(path)
 197     full_path = os.path.normpath(os.path.join(conformance_path, path))
 198
 199     if not os.path.exists(full_path):
 200       raise Exception('The WebGL conformance test path specified ' +
 201         'does not exist: ' + full_path)
 202
 203     with open(full_path, 'r') as f:
 204       for line in f:
 205         line = line.strip()
 206
 207         if not line:
 208           continue
 209
 210         if line.startswith('//') or line.startswith('#'):
 211           continue
 212
 213         line_tokens = line.split(' ')
 214         test_name = line_tokens[-1]
 215
 216         i = 0
 217         min_version = None
 218         while i < len(line_tokens):
 219           token = line_tokens[i]
 220           if token == '--min-version':
 221             i += 1
 222             min_version = line_tokens[i]
 223           i += 1
 224
 225         min_version_to_compare = min_version or folder_min_version
 226
 227         if (min_version_to_compare and
 228             _CompareVersion(version, min_version_to_compare) < 0):
 229           continue
 230
 231         if (webgl2_only and (not ('.txt' in test_name)) and
 232             ((not min_version_to_compare) or
 233              (not min_version_to_compare.startswith('2')))):
 234           continue
 235
 236         if '.txt' in test_name:
 237           include_path = os.path.join(current_dir, test_name)
 238           # We only check min-version >= 2.0.0 for the top level list.
 239           test_paths += WebglConformance._ParseTests(
 240               include_path, version, webgl2_only, min_version_to_compare)
 241         else:
 242           test = os.path.join(current_dir, test_name)
 243           test_paths.append(test)
 244
 245     return test_paths