content/test/gpu/gpu_tests/webgl_conformance_expectations.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import os
   5
   6 from gpu_test_expectations import GpuTestExpectations
   7
   8 # See the GpuTestExpectations class for documentation.
   9
  10 class WebGLConformanceExpectations(GpuTestExpectations):
  11   def __init__(self, conformance_path):
  12     self.conformance_path = conformance_path
  13     super(WebGLConformanceExpectations, self).__init__()
  14
  15   def Fail(self, pattern, condition=None, bug=None):
  16     self.CheckPatternIsValid(pattern)
  17     GpuTestExpectations.Fail(self, pattern, condition, bug)
  18
  19   def Skip(self, pattern, condition=None, bug=None):
  20     self.CheckPatternIsValid(pattern)
  21     GpuTestExpectations.Skip(self, pattern, condition, bug)
  22
  23   def CheckPatternIsValid(self, pattern):
  24     # Look for basic wildcards.
  25     if not '*' in pattern:
  26       full_path = os.path.normpath(os.path.join(self.conformance_path, pattern))
  27       if not os.path.exists(full_path):
  28         raise Exception('The WebGL conformance test path specified in' +
  29           'expectation does not exist: ' + full_path)
  30
  31   def SetExpectations(self):
  32     # Fails on all platforms
  33     self.Fail('deqp/data/gles2/shaders/functions.html',
  34         bug=478572)
  35     self.Fail('deqp/data/gles2/shaders/preprocessor.html',
  36         bug=478572)
  37     self.Fail('deqp/data/gles2/shaders/scoping.html',
  38         bug=478572)
  39     self.Fail('conformance/misc/expando-loss.html',
  40         bug=485634)
  41
  42     # Win failures
  43     self.Fail('conformance/glsl/bugs/' +
  44               'pow-of-small-constant-in-user-defined-function.html',
  45         ['win'], bug=485641)
  46     self.Fail('conformance/glsl/bugs/sampler-struct-function-arg.html',
  47         ['win'], bug=485642)
  48
  49     # Win7 / Intel failures
  50     self.Fail('conformance/rendering/gl-scissor-test.html',
  51         ['win7', 'intel'], bug=314997)
  52     self.Fail('conformance/context/premultiplyalpha-test.html',
  53         ['win7', 'intel'])
  54     self.Fail('conformance/textures/misc/copy-tex-image-and-sub-image-2d.html',
  55         ['win7', 'intel'])
  56     self.Fail('conformance/rendering/gl-viewport-test.html',
  57         ['win7', 'intel'], bug=372511)
  58
  59     # Win / AMD flakiness seen on new tryservers.
  60     # It's unfortunate that this suppression needs to be so broad, but
  61     # basically any test that uses readPixels is potentially flaky, and
  62     # it's infeasible to suppress individual failures one by one.
  63     self.Flaky('conformance/*', ['win', ('amd', 0x6779)], bug=491419)
  64
  65     # Win / AMD D3D9 failures
  66     self.Fail('conformance/textures/misc/texparameter-test.html',
  67         ['win', 'amd', 'd3d9'], bug=839) # angle bug ID
  68     self.Fail('conformance/extensions/angle-instanced-arrays.html',
  69         ['win', 'amd', 'd3d9'], bug=475095)
  70     self.Fail('conformance/rendering/more-than-65536-indices.html',
  71         ['win', 'amd', 'd3d9'], bug=475095)
  72
  73     # Win / D3D9 failures
  74     # Skipping these tests because they're causing assertion failures.
  75     self.Skip('conformance/extensions/oes-texture-float-with-canvas.html',
  76         ['win', 'd3d9'], bug=896) # angle bug ID
  77     self.Skip('conformance/extensions/oes-texture-half-float-with-canvas.html',
  78         ['win', 'd3d9'], bug=896) # angle bug ID
  79
  80     # Win / OpenGL failures
  81     self.Fail('conformance/attribs/gl-disabled-vertex-attrib.html',
  82         ['win', 'opengl'], bug=1007) # angle bug ID
  83     self.Fail('conformance/attribs/gl-disabled-vertex-attrib.html',
  84         ['win', 'opengl'], bug=1007) # angle bug ID
  85     self.Fail('conformance/canvas/canvas-test.html',
  86         ['win', 'opengl'], bug=1007) # angle bug ID
  87     self.Fail('conformance/canvas/draw-webgl-to-canvas-test.html',
  88         ['win', 'opengl'], bug=1007) # angle bug ID
  89     self.Fail('conformance/canvas/' +
  90         'draw-static-webgl-to-multiple-canvas-test.html',
  91         ['win', 'opengl'], bug=1007) # angle bug ID
  92     self.Fail('conformance/canvas/to-data-url-test.html',
  93         ['win', 'opengl'], bug=1007) # angle bug ID
  94     self.Fail('conformance/context/' +
  95         'context-attribute-preserve-drawing-buffer.html',
  96         ['win', 'opengl'], bug=1007) # angle bug ID
  97     self.Fail('conformance/context/'+
  98         'context-attributes-alpha-depth-stencil-antialias.html',
  99         ['win', 'opengl'], bug=1007) # angle bug ID
 100     self.Fail('conformance/context/premultiplyalpha-test.html',
 101         ['win', 'opengl'], bug=1007) # angle bug ID
 102     self.Fail('conformance/extensions/ext-sRGB.html',
 103         ['win', 'opengl'], bug=1007) # angle bug ID
 104     self.Fail('conformance/extensions/oes-texture-float-with-canvas.html',
 105         ['win', 'opengl'], bug=1007) # angle bug ID
 106     self.Fail('conformance/extensions/oes-texture-float.html',
 107         ['win', 'opengl'], bug=1007) # angle bug ID
 108     self.Fail('conformance/extensions/oes-texture-half-float.html',
 109         ['win', 'opengl'], bug=1007) # angle bug ID
 110     self.Fail('conformance/extensions/oes-texture-half-float-with-canvas.html',
 111         ['win', 'opengl'], bug=1007) # angle bug ID
 112     self.Fail('conformance/glsl/variables/gl-pointcoord.html',
 113         ['win', 'opengl'], bug=1007) # angle bug ID
 114     self.Fail('conformance/reading/read-pixels-pack-alignment.html',
 115         ['win', 'opengl'], bug=1007) # angle bug ID
 116     self.Fail('conformance/renderbuffers/framebuffer-object-attachment.html',
 117         ['win', 'opengl'], bug=1007) # angle bug ID
 118     self.Fail('conformance/rendering/point-size.html',
 119         ['win', 'opengl'], bug=1007) # angle bug ID
 120     self.Fail('conformance/rendering/' +
 121         'point-with-gl-pointcoord-in-fragment-shader.html',
 122         ['win', 'opengl'], bug=1007) # angle bug ID
 123     self.Fail('conformance/textures/misc/gl-pixelstorei.html',
 124         ['win', 'opengl'], bug=1007) # angle bug ID
 125     self.Fail('conformance/textures/misc/tex-image-canvas-corruption.html',
 126         ['win', 'opengl'], bug=1007) # angle bug ID
 127     self.Fail('deqp/data/gles2/shaders/conditionals.html',
 128         ['win', 'opengl'], bug=1007) # angle bug ID
 129     self.Fail('deqp/data/gles2/shaders/conversions.html',
 130         ['win', 'opengl'], bug=1007) # angle bug ID
 131     self.Fail('deqp/data/gles2/shaders/qualification_order.html',
 132         ['win', 'opengl'], bug=1007) # angle bug ID
 133     self.Fail('deqp/data/gles2/shaders/linkage.html',
 134         ['win', 'opengl'], bug=1007) # angle bug ID
 135
 136     # Win / OpenGL / AMD failures
 137     self.Fail('conformance/extensions/oes-texture-half-float-linear.html',
 138         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 139     self.Fail('conformance/extensions/' +
 140         'oes-texture-half-float-with-image-data.html',
 141         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 142     self.Fail('conformance/extensions/oes-texture-half-float-with-image.html',
 143         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 144     self.Fail('conformance/extensions/oes-texture-half-float-with-video.html',
 145         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 146     self.Fail('conformance/glsl/misc/shader-struct-scope.html',
 147         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 148     self.Fail('conformance/textures/misc/tex-sub-image-2d.html',
 149         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 150     self.Fail('conformance/textures/misc/texture-formats-test.html',
 151         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 152     self.Fail('conformance/textures/misc/texture-npot.html',
 153         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 154     self.Fail('conformance/more/functions/copyTexImage2D.html',
 155         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 156     self.Fail('conformance/more/functions/copyTexSubImage2D.html',
 157         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 158     self.Fail('conformance/more/functions/uniformf.html',
 159         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 160     self.Fail('conformance/more/functions/uniformfBadArgs.html',
 161         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 162     self.Fail('conformance/more/functions/uniformfArrayLen1.html',
 163         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 164     self.Fail('conformance/more/functions/uniformi.html',
 165         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 166     self.Fail('conformance/more/functions/uniformiBadArgs.html',
 167         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 168     self.Fail('conformance/more/functions/uniformMatrix.html',
 169         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 170     self.Fail('conformance/more/functions/uniformMatrixBadArgs.html',
 171         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 172     self.Fail('deqp/data/gles2/shaders/swizzles.html',
 173         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 174     self.Fail('conformance/glsl/misc/' +
 175         'shader-with-array-of-structs-containing-arrays.html',
 176         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 177     self.Fail('conformance/glsl/misc/' +
 178         'shader-with-similar-uniform-array-names.html',
 179         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 180     self.Fail('conformance/glsl/misc/shaders-with-invariance.html',
 181         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 182     self.Fail('conformance/glsl/misc/struct-nesting-of-variable-names.html',
 183         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 184     self.Fail('conformance/rendering/gl-scissor-test.html',
 185         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 186     self.Fail('conformance/state/gl-get-calls.html',
 187         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 188     self.Fail('conformance/uniforms/gl-uniform-arrays.html',
 189         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 190     self.Fail('deqp/data/gles2/shaders/constant_expressions.html',
 191         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 192     self.Fail('deqp/data/gles2/shaders/constants.html',
 193         ['win', 'amd', 'opengl'], bug=1007) # angle bug ID
 194
 195     # Skip all WebGL CTS on OpenGL+Intel
 196     self.Skip('*', ['win', 'opengl', 'intel'], bug=1007) # angle bug ID
 197
 198     # Mac failures
 199     self.Fail('conformance/glsl/misc/shaders-with-invariance.html',
 200         ['mac'], bug=421710)
 201
 202     # Mac / Intel failures
 203     # Radar 13499466
 204     self.Fail('conformance/limits/gl-max-texture-dimensions.html',
 205         ['mac', 'intel'], bug=225642)
 206     # Radar 13499623
 207     self.Fail('conformance/textures/misc/texture-size.html',
 208         ['mac', 'intel'], bug=225642)
 209
 210     # Mac / Intel HD 3000 failures
 211     self.Skip('conformance/ogles/GL/control_flow/control_flow_009_to_010.html',
 212         ['mac', ('intel', 0x116)], bug=322795)
 213     # Radar 13499677
 214     self.Fail('conformance/glsl/functions/' +
 215         'glsl-function-smoothstep-gentype.html',
 216         ['mac', ('intel', 0x116)], bug=225642)
 217     self.Fail('conformance/extensions/webgl-draw-buffers.html',
 218         ['mac', ('intel', 0x116)], bug=369349)
 219
 220     # Mac 10.8 / Intel HD 3000 failures
 221     self.Fail('conformance/rendering/gl-scissor-test.html',
 222         ['mountainlion', ('intel', 0x116)], bug=314997)
 223     self.Fail('conformance/ogles/GL/operators/operators_009_to_016.html',
 224         ['mountainlion', ('intel', 0x116)], bug=322795)
 225
 226     # Mac 10.9 / Intel HD 3000 failures
 227     self.Fail('conformance/ogles/GL/operators/operators_009_to_016.html',
 228         ['mavericks', ('intel', 0x116)], bug=417415)
 229     self.Fail('conformance/rendering/gl-scissor-test.html',
 230         ['mavericks', ('intel', 0x116)], bug=417415)
 231
 232     # Mac Retina failures
 233     self.Fail(
 234         'conformance/glsl/bugs/array-of-struct-with-int-first-position.html',
 235         ['mac', ('nvidia', 0xfd5), ('nvidia', 0xfe9)], bug=368912)
 236
 237     # Mac / AMD Failures
 238     self.Fail('deqp/data/gles2/shaders/conversions.html',
 239         ['mac', 'amd'], bug=478572)
 240
 241     # Mac 10.10 / AMD Failures
 242     self.Flaky('conformance/rendering/many-draw-calls.html',
 243         ['mac', 'yosemite', 'amd', 'debug'], bug=505814)
 244
 245     # Mac 10.8 / ATI failures
 246     self.Fail(
 247         'conformance/rendering/' +
 248         'point-with-gl-pointcoord-in-fragment-shader.html',
 249         ['mountainlion', 'amd'])
 250
 251     # Mac 10.7 / Intel failures
 252     self.Skip('conformance/glsl/functions/glsl-function-asin.html',
 253         ['lion', 'intel'])
 254     self.Skip('conformance/glsl/functions/glsl-function-dot.html',
 255         ['lion', 'intel'])
 256     self.Skip('conformance/glsl/functions/glsl-function-faceforward.html',
 257         ['lion', 'intel'])
 258     self.Skip('conformance/glsl/functions/glsl-function-length.html',
 259         ['lion', 'intel'])
 260     self.Skip('conformance/glsl/functions/glsl-function-normalize.html',
 261         ['lion', 'intel'])
 262     self.Skip('conformance/glsl/functions/glsl-function-reflect.html',
 263         ['lion', 'intel'])
 264     self.Skip('conformance/rendering/line-loop-tri-fan.html',
 265         ['lion', 'intel'])
 266     self.Skip('conformance/ogles/GL/control_flow/control_flow_001_to_008.html',
 267         ['lion', 'intel'], bug=345575)
 268     self.Skip('conformance/ogles/GL/dot/dot_001_to_006.html',
 269         ['lion', 'intel'], bug=323736)
 270     self.Skip('conformance/ogles/GL/faceforward/faceforward_001_to_006.html',
 271         ['lion', 'intel'], bug=323736)
 272     self.Skip('conformance/ogles/GL/length/length_001_to_006.html',
 273         ['lion', 'intel'], bug=323736)
 274     self.Skip('conformance/ogles/GL/normalize/normalize_001_to_006.html',
 275         ['lion', 'intel'], bug=323736)
 276     self.Skip('conformance/ogles/GL/reflect/reflect_001_to_006.html',
 277         ['lion', 'intel'], bug=323736)
 278     self.Skip('conformance/ogles/GL/refract/refract_001_to_006.html',
 279         ['lion', 'intel'], bug=323736)
 280     self.Skip('conformance/ogles/GL/tan/tan_001_to_006.html',
 281         ['lion', 'intel'], bug=323736)
 282     # Two flaky tests.
 283     self.Fail('conformance/ogles/GL/functions/functions_049_to_056.html',
 284         ['lion', 'intel'], bug=393331)
 285     self.Fail('conformance/extensions/webgl-compressed-texture-size-limit.html',
 286         ['lion', 'intel'], bug=393331)
 287
 288     # Linux failures
 289     # NVIDIA
 290     self.Fail('conformance/textures/misc/default-texture.html',
 291         ['linux', ('nvidia', 0x104a)], bug=422152)
 292     # AMD Radeon 5450
 293     self.Fail('conformance/programs/program-test.html',
 294         ['linux', ('amd', 0x68f9)], bug=436212)
 295     self.Fail('conformance/rendering/multisample-corruption.html',
 296         ['linux', ('amd', 0x68f9)], bug=436212)
 297     self.Fail('conformance/textures/misc/default-texture.html',
 298         ['linux', ('amd', 0x68f9)], bug=436212)
 299     self.Fail('conformance/textures/video/' +
 300         'tex-image-and-sub-image-2d-with-video-rgb-rgb-unsigned_byte.html',
 301         ['linux', ('amd', 0x68f9)], bug=436212)
 302     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 303         'video-rgb-rgb-unsigned_short_5_6_5.html',
 304         ['linux', ('amd', 0x68f9)], bug=436212)
 305     self.Fail('conformance/textures/video/' +
 306         'tex-image-and-sub-image-2d-with-video-rgba-rgba-unsigned_byte.html',
 307         ['linux', ('amd', 0x68f9)], bug=436212)
 308     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 309         'video-rgba-rgba-unsigned_short_4_4_4_4.html',
 310         ['linux', ('amd', 0x68f9)], bug=436212)
 311     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 312         'video-rgba-rgba-unsigned_short_5_5_5_1.html',
 313         ['linux', ('amd', 0x68f9)], bug=436212)
 314     self.Fail('conformance/textures/webgl_canvas/tex-image-and-sub-image-2d-' +
 315         'with-webgl-canvas-rgb-rgb-unsigned_byte.html',
 316         ['linux', ('amd', 0x68f9)], bug=436212)
 317     self.Fail('conformance/textures/webgl_canvas/tex-image-and-sub-image-2d-' +
 318         'with-webgl-canvas-rgb-rgb-unsigned_short_5_6_5.html',
 319         ['linux', ('amd', 0x68f9)], bug=436212)
 320     self.Fail('conformance/textures/webgl_canvas/tex-image-and-sub-image-2d-' +
 321         'with-webgl-canvas-rgba-rgba-unsigned_byte.html',
 322         ['linux', ('amd', 0x68f9)], bug=436212)
 323     self.Fail('conformance/textures/webgl_canvas/tex-image-and-sub-image-2d-' +
 324         'with-webgl-canvas-rgba-rgba-unsigned_short_4_4_4_4.html',
 325         ['linux', ('amd', 0x68f9)], bug=436212)
 326     self.Fail('conformance/textures/webgl_canvas/tex-image-and-sub-image-2d-' +
 327         'with-webgl-canvas-rgba-rgba-unsigned_short_5_5_5_1.html',
 328         ['linux', ('amd', 0x68f9)], bug=436212)
 329     self.Fail('conformance/textures/misc/texture-mips.html',
 330         ['linux', ('amd', 0x68f9)], bug=436212)
 331     self.Fail('conformance/textures/misc/texture-npot-video.html',
 332         ['linux', ('amd', 0x68f9)], bug=436212)
 333     self.Fail('conformance/textures/misc/texture-size.html',
 334         ['linux', ('amd', 0x68f9)], bug=436212)
 335     self.Fail('conformance/more/functions/copyTexSubImage2D.html',
 336         ['linux', ('amd', 0x68f9)], bug=436212)
 337     self.Fail('conformance/more/functions/drawArraysOutOfBounds.html',
 338         ['linux', ('amd', 0x68f9)], bug=436212)
 339     self.Fail('conformance/more/functions/texImage2DHTML.html',
 340         ['linux', ('amd', 0x68f9)], bug=436212)
 341     self.Fail('conformance/more/functions/texSubImage2DHTML.html',
 342         ['linux', ('amd', 0x68f9)], bug=436212)
 343     # AMD Radeon 6450
 344     self.Fail('conformance/extensions/angle-instanced-arrays.html',
 345         ['linux', ('amd', 0x6779)], bug=479260)
 346     self.Fail('conformance/extensions/ext-texture-filter-anisotropic.html',
 347         ['linux', ('amd', 0x6779)], bug=436212)
 348     self.Fail('conformance/glsl/misc/shader-struct-scope.html',
 349         ['linux', ('amd', 0x6779)], bug=436212)
 350     self.Fail('conformance/glsl/misc/struct-nesting-of-variable-names.html',
 351         ['linux', ('amd', 0x6779)], bug=436212)
 352     self.Fail('conformance/rendering/point-size.html',
 353         ['linux', ('amd', 0x6779)], bug=436212)
 354     self.Fail('conformance/textures/misc/texture-sub-image-cube-maps.html',
 355         ['linux', ('amd', 0x6779)], bug=436212)
 356     self.Fail('conformance/more/functions/uniformf.html',
 357         ['linux', ('amd', 0x6779)], bug=436212)
 358     self.Fail('conformance/glsl/misc/shaders-with-invariance.html',
 359         ['linux', ('amd', 0x6779)], bug=479952)
 360     self.Fail('conformance/textures/misc/texture-mips.html',
 361         ['linux', ('amd', 0x6779)], bug=479981)
 362     self.Fail('conformance/textures/misc/texture-size-cube-maps.html',
 363         ['linux', ('amd', 0x6779)], bug=479983)
 364     self.Fail('conformance/uniforms/uniform-default-values.html',
 365         ['linux', ('amd', 0x6779)], bug=482013)
 366
 367     # Android failures
 368     self.Fail('deqp/data/gles2/shaders/constants.html',
 369         ['android'], bug=478572)
 370     self.Fail('deqp/data/gles2/shaders/conversions.html',
 371         ['android'], bug=478572)
 372     self.Fail('deqp/data/gles2/shaders/declarations.html',
 373         ['android'], bug=478572)
 374     self.Fail('deqp/data/gles2/shaders/linkage.html',
 375         ['android'], bug=478572)
 376     # The following WebView crashes are causing problems with further
 377     # tests in the suite, so skip them for now.
 378     self.Skip('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 379         'video-rgb-rgb-unsigned_byte.html',
 380         ['android', 'android-webview-shell'], bug=352645)
 381     self.Skip('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 382         'video-rgb-rgb-unsigned_short_5_6_5.html',
 383         ['android', 'android-webview-shell'], bug=352645)
 384     self.Skip('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 385         'video-rgba-rgba-unsigned_byte.html',
 386         ['android', 'android-webview-shell'], bug=352645)
 387     self.Skip('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 388         'video-rgba-rgba-unsigned_short_4_4_4_4.html',
 389         ['android', 'android-webview-shell'], bug=352645)
 390     self.Skip('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 391         'video-rgba-rgba-unsigned_short_5_5_5_1.html',
 392         ['android', 'android-webview-shell'], bug=352645)
 393     self.Skip('conformance/textures/misc/texture-npot-video.html',
 394         ['android', 'android-webview-shell'], bug=352645)
 395     # Recent regressions have caused these to fail on multiple devices
 396     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 397         'video-rgb-rgb-unsigned_byte.html',
 398         ['android', 'android-content-shell'], bug=499555)
 399     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 400         'video-rgba-rgba-unsigned_byte.html',
 401         ['android', 'android-content-shell'], bug=499555)
 402     # These are failing on the Nexus 5 and 6
 403     self.Fail('conformance/extensions/oes-texture-float-with-canvas.html',
 404               ['android', 'qualcomm'], bug=499555)
 405     # This crashes in Android WebView on the Nexus 6, preventing the
 406     # suite from running further. Rather than add multiple
 407     # suppressions, skip it until it's passing at least in content
 408     # shell.
 409     self.Skip('conformance/extensions/oes-texture-float-with-video.html',
 410               ['android', 'qualcomm'], bug=499555)
 411     # Nexus 6 failures only
 412     self.Fail('conformance/context/' +
 413               'context-attributes-alpha-depth-stencil-antialias.html',
 414               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 415     self.Fail('conformance/context/premultiplyalpha-test.html',
 416               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 417     self.Fail('conformance/extensions/oes-texture-float-with-image-data.html',
 418               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 419     self.Fail('conformance/extensions/oes-texture-float-with-image.html',
 420               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 421     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 422         'video-rgb-rgb-unsigned_short_5_6_5.html',
 423         ['android', 'android-content-shell',
 424          ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 425     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 426         'video-rgba-rgba-unsigned_short_4_4_4_4.html',
 427         ['android', 'android-content-shell',
 428          ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 429     self.Fail('conformance/textures/video/tex-image-and-sub-image-2d-with-' +
 430         'video-rgba-rgba-unsigned_short_5_5_5_1.html',
 431         ['android', 'android-content-shell',
 432          ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 433     # bindBufferBadArgs is causing the GPU thread to crash, taking
 434     # down the WebView shell, causing the next test to fail and
 435     # subsequent tests to be aborted.
 436     self.Skip('conformance/more/functions/bindBufferBadArgs.html',
 437               ['android', 'android-webview-shell',
 438                ('qualcomm', 'Adreno (TM) 420')], bug=499874)
 439     self.Fail('conformance/rendering/gl-scissor-test.html',
 440               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 441     self.Fail('conformance/textures/misc/' +
 442               'copy-tex-image-and-sub-image-2d.html',
 443               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 444     self.Fail('conformance/textures/misc/' +
 445               'tex-image-and-sub-image-2d-with-array-buffer-view.html',
 446               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 447     self.Fail('conformance/textures/canvas/*',
 448               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 449     self.Fail('conformance/textures/image_data/*',
 450               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 451     self.Fail('conformance/textures/image/*',
 452               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 453     self.Fail('conformance/textures/webgl_canvas/*',
 454               ['android', ('qualcomm', 'Adreno (TM) 420')], bug=499555)
 455     # Nexus 9 failures
 456     self.Skip('conformance/extensions/oes-texture-float-with-video.html',
 457               ['android', 'nvidia'], bug=499555) # flaky
 458
 459     # The following test is very slow and therefore times out on Android bot.
 460     self.Skip('conformance/rendering/multisample-corruption.html',
 461         ['android'])
 462
 463     # ChromeOS: affecting all devices.
 464     self.Fail('conformance/extensions/webgl-depth-texture.html',
 465         ['chromeos'], bug=382651)
 466
 467     # ChromeOS: all Intel except for pinetrail (stumpy, parrot, peppy,...)
 468     # We will just include pinetrail here for now as we don't want to list
 469     # every single Intel device ID.
 470     self.Fail('conformance/glsl/misc/empty_main.vert.html',
 471         ['chromeos', 'intel'], bug=375556)
 472     self.Fail('conformance/glsl/misc/gl_position_unset.vert.html',
 473         ['chromeos', 'intel'], bug=375556)
 474     self.Fail('conformance/glsl/misc/shaders-with-varyings.html',
 475         ['chromeos', 'intel'], bug=375556)
 476     self.Fail('conformance/renderbuffers/framebuffer-object-attachment.html',
 477         ['chromeos', 'intel'], bug=375556)
 478     self.Fail('conformance/textures/misc/texture-size-limit.html',
 479         ['chromeos', 'intel'], bug=385361)
 480
 481     # ChromeOS: pinetrail (alex, mario, zgb).
 482     self.Fail('conformance/attribs/gl-vertex-attrib-render.html',
 483         ['chromeos', ('intel', 0xa011)], bug=375554)
 484     self.Fail('conformance/glsl/functions/glsl-function-atan-xy.html',
 485         ['chromeos', ('intel', 0xa011)], bug=375554)
 486     self.Fail('conformance/glsl/functions/glsl-function-cos.html',
 487         ['chromeos', ('intel', 0xa011)], bug=375554)
 488     self.Fail('conformance/glsl/functions/glsl-function-sin.html',
 489         ['chromeos', ('intel', 0xa011)], bug=375554)
 490     self.Fail('conformance/glsl/misc/empty_main.vert.html',
 491         ['chromeos', ('intel', 0xa011)], bug=375554)
 492     self.Fail('conformance/glsl/misc/gl_position_unset.vert.html',
 493         ['chromeos', ('intel', 0xa011)], bug=375554)
 494     self.Fail('conformance/glsl/misc/shaders-with-varyings.html',
 495         ['chromeos', ('intel', 0xa011)], bug=375554)
 496     self.Fail('conformance/glsl/variables/gl-frontfacing.html',
 497         ['chromeos', ('intel', 0xa011)], bug=375554)
 498     self.Fail('conformance/limits/gl-max-texture-dimensions.html',
 499         ['chromeos', ('intel', 0xa011)], bug=375554)
 500     self.Fail('conformance/ogles/GL/acos/acos_001_to_006.html',
 501         ['chromeos', ('intel', 0xa011)], bug=375554)
 502     self.Fail('conformance/ogles/GL/asin/asin_001_to_006.html',
 503         ['chromeos', ('intel', 0xa011)], bug=375554)
 504     self.Fail('conformance/ogles/GL/atan/atan_001_to_008.html',
 505         ['chromeos', ('intel', 0xa011)], bug=375554)
 506     self.Fail('conformance/ogles/GL/build/build_009_to_016.html',
 507         ['chromeos', ('intel', 0xa011)], bug=378938)
 508     self.Fail('conformance/ogles/GL/control_flow/control_flow_001_to_008.html',
 509         ['chromeos', ('intel', 0xa011)], bug=375554)
 510     self.Fail('conformance/ogles/GL/cos/cos_001_to_006.html',
 511         ['chromeos', ('intel', 0xa011)], bug=375554)
 512     self.Fail('conformance/ogles/GL/discard/discard_001_to_002.html',
 513         ['chromeos', ('intel', 0xa011)], bug=375554)
 514     self.Fail('conformance/ogles/GL/functions/functions_001_to_008.html',
 515         ['chromeos', ('intel', 0xa011)], bug=375554)
 516     self.Fail('conformance/ogles/GL/functions/functions_065_to_072.html',
 517         ['chromeos', ('intel', 0xa011)], bug=375554)
 518     self.Fail('conformance/ogles/GL/functions/functions_081_to_088.html',
 519         ['chromeos', ('intel', 0xa011)], bug=375554)
 520     self.Fail('conformance/ogles/GL/functions/functions_097_to_104.html',
 521         ['chromeos', ('intel', 0xa011)], bug=375554)
 522     self.Fail('conformance/ogles/GL/functions/functions_105_to_112.html',
 523         ['chromeos', ('intel', 0xa011)], bug=375554)
 524     self.Fail('conformance/ogles/GL/functions/functions_113_to_120.html',
 525         ['chromeos', ('intel', 0xa011)], bug=375554)
 526     self.Fail('conformance/ogles/GL/functions/functions_121_to_126.html',
 527         ['chromeos', ('intel', 0xa011)], bug=375554)
 528     self.Fail(
 529         'conformance/ogles/GL/gl_FrontFacing/gl_FrontFacing_001_to_001.html',
 530         ['chromeos', ('intel', 0xa011)], bug=375554)
 531     self.Fail('conformance/ogles/GL/log/log_001_to_008.html',
 532         ['chromeos', ('intel', 0xa011)], bug=375554)
 533     self.Fail('conformance/ogles/GL/log2/log2_001_to_008.html',
 534         ['chromeos', ('intel', 0xa011)], bug=375554)
 535     self.Fail('conformance/ogles/GL/normalize/normalize_001_to_006.html',
 536         ['chromeos', ('intel', 0xa011)], bug=375554)
 537     self.Fail('conformance/ogles/GL/sin/sin_001_to_006.html',
 538         ['chromeos', ('intel', 0xa011)], bug=375554)
 539     self.Fail('conformance/rendering/point-size.html',
 540         ['chromeos', ('intel', 0xa011)], bug=375554)
 541     self.Fail('conformance/rendering/polygon-offset.html',
 542         ['chromeos', ('intel', 0xa011)], bug=375554)
 543     self.Fail('conformance/textures/misc/texture-mips.html',
 544         ['chromeos', ('intel', 0xa011)], bug=375554)
 545     self.Fail('conformance/textures/misc/texture-npot.html',
 546         ['chromeos', ('intel', 0xa011)], bug=375554)
 547     self.Fail('conformance/textures/misc/texture-npot-video.html',
 548         ['chromeos', ('intel', 0xa011)], bug=375554)
 549     self.Fail('conformance/textures/misc/texture-size.html',
 550         ['chromeos', ('intel', 0xa011)], bug=375554)
 551     self.Fail('conformance/textures/misc/texture-size-limit.html',
 552         ['chromeos', ('intel', 0xa011)], bug=375554)
 553     self.Fail('conformance/uniforms/gl-uniform-arrays.html',
 554         ['chromeos', ('intel', 0xa011)], bug=375554)
 555     self.Skip('conformance/uniforms/uniform-default-values.html',
 556         ['chromeos', ('intel', 0xa011)], bug=375554)
 557
 558     # Flaky on Mac & Linux
 559     self.Fail('conformance/textures/misc/texture-upload-size.html',
 560         ['mac'], bug=436493)
 561     self.Fail('conformance/textures/misc/texture-upload-size.html',
 562         ['linux'], bug=436493)
 563
 564     ##############################################################
 565     # WEBGL 2 TESTS FAILURES
 566     ##############################################################
 567
 568     self.Fail('deqp/data/gles3/shaders/arrays.html', bug=483282)
 569     self.Fail('deqp/data/gles3/shaders/constant_expressions.html', bug=483282)
 570     self.Fail('deqp/data/gles3/shaders/constants.html', bug=483282)
 571     self.Skip('deqp/data/gles3/shaders/conversions.html', bug=483282)
 572     self.Skip('deqp/data/gles3/shaders/functions.html', bug=483282)
 573     self.Skip('deqp/data/gles3/shaders/linkage.html', bug=483282)
 574     self.Fail('deqp/data/gles3/shaders/preprocessor.html', bug=483282)
 575     self.Fail('deqp/data/gles3/shaders/qualification_order.html', bug=483282)
 576     self.Fail('deqp/data/gles3/shaders/scoping.html', bug=483282)
 577
 578     self.Fail('deqp/framework/opengl/simplereference/referencecontext.html',
 579         bug=483282)
 580
 581     self.Skip('deqp/functional/gles3/attriblocation.html', bug=483282)
 582     self.Skip('deqp/functional/gles3/buffercopy.html', bug=483282)
 583     self.Skip('deqp/functional/gles3/draw.html', bug=483282)
 584     self.Fail('deqp/functional/gles3/fbocompleteness.html', bug=483282)
 585     self.Skip('deqp/functional/gles3/fbomultisample.html', bug=483282)
 586     self.Fail('deqp/functional/gles3/fbostencilbuffer.html', bug=483282)
 587     self.Skip('deqp/functional/gles3/fragmentoutput.html', bug=483282)
 588     self.Skip('deqp/functional/gles3/framebufferblit.html', bug=483282)
 589     self.Skip('deqp/functional/gles3/instancedrendering.html', bug=483282)
 590     self.Fail('deqp/functional/gles3/lifetime.html', bug=483282)
 591     self.Skip('deqp/functional/gles3/multisample.html', bug=483282)
 592     self.Skip('deqp/functional/gles3/negativebufferapi.html', bug=483282)
 593     self.Skip('deqp/functional/gles3/negativefragmentapi.html', bug=483282)
 594     self.Skip('deqp/functional/gles3/negativeshaderapi.html', bug=483282)
 595     self.Skip('deqp/functional/gles3/negativestateapi.html', bug=483282)
 596     self.Skip('deqp/functional/gles3/negativetextureapi.html', bug=483282)
 597     self.Skip('deqp/functional/gles3/negativevertexarrayapi.html', bug=483282)
 598     self.Skip('deqp/functional/gles3/pixelbufferobject.html', bug=483282)
 599     self.Fail('deqp/functional/gles3/primitiverestart.html', bug=483282)
 600     self.Skip('deqp/functional/gles3/samplerobject.html', bug=483282)
 601     self.Fail('deqp/functional/gles3/textureformat.html', bug=483282)
 602     self.Skip('deqp/functional/gles3/textureshadow.html', bug=483282)
 603     self.Fail('deqp/functional/gles3/texturewrap.html', bug=483282)
 604     self.Skip('deqp/functional/gles3/transformfeedback.html', bug=483282)
 605     self.Fail('deqp/functional/gles3/uniformapi.html', bug=483282)
 606     self.Skip('deqp/functional/gles3/uniformbuffers.html', bug=483282)
 607     self.Skip('deqp/functional/gles3/vertexarrays.html', bug=483282)
 608
 609     self.Fail('conformance2/attribs/gl-vertex-attrib.html', bug=483282)
 610     self.Fail('conformance2/attribs/gl-vertex-attrib-i-render.html', bug=483282)
 611     self.Fail('conformance2/attribs/gl-vertexattribipointer.html', bug=483282)
 612     self.Fail('conformance2/attribs/gl-vertexattribipointer-offsets.html',
 613         bug=483282)
 614     self.Fail('conformance2/buffers/buffer-copying-contents.html', bug=483282)
 615     self.Fail('conformance2/buffers/buffer-copying-restrictions.html',
 616         bug=483282)
 617     self.Fail('conformance2/buffers/buffer-type-restrictions.html', bug=483282)
 618     self.Fail('conformance2/buffers/getBufferSubData.html', bug=483282)
 619     self.Fail('conformance2/context/constants-and-properties-2.html',
 620         bug=483282)
 621
 622     self.Fail('conformance2/glsl3/array-complex-indexing.html', bug=483282)
 623     self.Fail('conformance2/glsl3/frag-depth.html', bug=483282)
 624     self.Fail('conformance2/glsl3/invalid-default-precision.html', bug=483282)
 625     self.Fail('conformance2/glsl3/sequence-operator-returns-non-constant.html',
 626         bug=483282)
 627     self.Fail('conformance2/glsl3/shader-with-1024-character-define.html',
 628         bug=483282)
 629     self.Fail('conformance2/glsl3/' +
 630         'shader-with-1024-character-identifier.frag.html',
 631         bug=483282)
 632     self.Fail('conformance2/glsl3/short-circuiting-in-loop-condition.html',
 633         bug=483282)
 634     self.Fail('conformance2/glsl3/ternary-operator-on-arrays-glsl3.html',
 635         bug=483282)
 636     self.Fail('conformance2/misc/instanceof-test.html', bug=483282)
 637     self.Fail('conformance2/query/query.html', bug=483282)
 638     self.Fail('conformance2/renderbuffers/framebuffer-test.html', bug=483282)
 639     self.Fail('conformance2/renderbuffers/invalidate-framebuffer.html',
 640         bug=483282)
 641     self.Fail('conformance2/renderbuffers/readbuffer.html', bug=483282)
 642     self.Fail('conformance2/rendering/draw-buffers.html', bug=483282)
 643     self.Fail('conformance2/samplers/samplers.html', bug=483282)
 644     self.Fail('conformance2/state/gl-enum-tests.html', bug=483282)
 645     self.Fail('conformance2/state/gl-get-calls.html', bug=483282)
 646     self.Fail('conformance2/state/gl-object-get-calls.html', bug=483282)
 647     self.Fail('conformance2/textures/canvas/*', bug=483282)
 648     self.Fail('conformance2/textures/image_data/*', bug=483282)
 649     self.Fail('conformance2/textures/image/*', bug=483282)
 650     self.Fail('conformance2/textures/svg_image/*', bug=483282)
 651     self.Fail('conformance2/textures/video/*', bug=483282)
 652     self.Fail('conformance2/textures/webgl_canvas/*', bug=483282)
 653     self.Fail('conformance2/textures/misc/gl-get-tex-parameter.html',
 654               bug=483282)
 655     self.Fail('conformance2/textures/misc/tex-input-validation.html',
 656               bug=483282)
 657     self.Fail('conformance2/textures/misc/tex-mipmap-levels.html', bug=483282)
 658     self.Fail('conformance2/textures/misc/tex-new-formats.html', bug=483282)
 659     self.Fail('conformance2/textures/misc/tex-storage-2d.html', bug=483282)
 660     self.Fail('conformance2/textures/misc/tex-storage-and-subimage-3d.html',
 661         bug=483282)
 662     self.Fail('conformance2/textures/misc/texture-npot.html', bug=483282)
 663     self.Fail('conformance2/transform_feedback/transform_feedback.html',
 664         bug=483282)
 665
 666     # Windows only.
 667     self.Fail('conformance2/glsl3/array-in-complex-expression.html',
 668         ['win'], bug=483282)
 669     self.Fail('conformance2/renderbuffers/framebuffer-object-attachment.html',
 670         ['win'], bug=1082) # angle bug ID
 671
 672     # Mac only.
 673     self.Fail('conformance2/glsl3/array-as-return-value.html',
 674         ['mac'], bug=483282)
 675     self.Fail('conformance2/glsl3/array-assign.html',
 676         ['mac'], bug=483282)
 677     self.Fail('conformance2/glsl3/array-equality.html',
 678         ['mac'], bug=483282)
 679     self.Fail('conformance2/glsl3/array-in-complex-expression.html',
 680         ['mac'], bug=483282)
 681     self.Fail('conformance2/renderbuffers/' +
 682         'multisampled-renderbuffer-initialization.html',
 683         ['mac'], bug=483282)
 684     self.Fail('conformance2/rendering/instanced-arrays.html',
 685         ['mac'], bug=483282)