test_shaders.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright 2015-2021 Arm Limited
   4 # SPDX-License-Identifier: Apache-2.0
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import sys
  19 import os
  20 import os.path
  21 import subprocess
  22 import tempfile
  23 import re
  24 import itertools
  25 import hashlib
  26 import shutil
  27 import argparse
  28 import codecs
  29 import json
  30 import multiprocessing
  31 import errno
  32 import platform
  33 from functools import partial
  34
  35 class Paths():
  36     def __init__(self, spirv_cross, glslang, spirv_as, spirv_val, spirv_opt):
  37         self.spirv_cross = spirv_cross
  38         self.glslang = glslang
  39         self.spirv_as = spirv_as
  40         self.spirv_val = spirv_val
  41         self.spirv_opt = spirv_opt
  42
  43 def remove_file(path):
  44     #print('Removing file:', path)
  45     os.remove(path)
  46
  47 def create_temporary(suff = ''):
  48     f, path = tempfile.mkstemp(suffix = suff)
  49     os.close(f)
  50     #print('Creating temporary:', path)
  51     return path
  52
  53 def parse_stats(stats):
  54     m = re.search('([0-9]+) work registers', stats)
  55     registers = int(m.group(1)) if m else 0
  56
  57     m = re.search('([0-9]+) uniform registers', stats)
  58     uniform_regs = int(m.group(1)) if m else 0
  59
  60     m_list = re.findall('(-?[0-9]+)\s+(-?[0-9]+)\s+(-?[0-9]+)', stats)
  61     alu_short = float(m_list[1][0]) if m_list else 0
  62     ls_short = float(m_list[1][1]) if m_list else 0
  63     tex_short = float(m_list[1][2]) if m_list else 0
  64     alu_long = float(m_list[2][0]) if m_list else 0
  65     ls_long = float(m_list[2][1]) if m_list else 0
  66     tex_long = float(m_list[2][2]) if m_list else 0
  67
  68     return (registers, uniform_regs, alu_short, ls_short, tex_short, alu_long, ls_long, tex_long)
  69
  70 def get_shader_type(shader):
  71     _, ext = os.path.splitext(shader)
  72     if ext == '.vert':
  73         return '--vertex'
  74     elif ext == '.frag':
  75         return '--fragment'
  76     elif ext == '.comp':
  77         return '--compute'
  78     elif ext == '.tesc':
  79         return '--tessellation_control'
  80     elif ext == '.tese':
  81         return '--tessellation_evaluation'
  82     elif ext == '.geom':
  83         return '--geometry'
  84     else:
  85         return ''
  86
  87 def get_shader_stats(shader):
  88     path = create_temporary()
  89
  90     p = subprocess.Popen(['malisc', get_shader_type(shader), '--core', 'Mali-T760', '-V', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
  91     stdout, stderr = p.communicate()
  92     remove_file(path)
  93
  94     if p.returncode != 0:
  95         print(stderr.decode('utf-8'))
  96         raise OSError('malisc failed')
  97     p.wait()
  98
  99     returned = stdout.decode('utf-8')
 100     return parse_stats(returned)
 101
 102 def print_msl_compiler_version():
 103     try:
 104         if platform.system() == 'Darwin':
 105             subprocess.check_call(['xcrun', '--sdk', 'iphoneos', 'metal', '--version'])
 106             print('... are the Metal compiler characteristics.\n')   # display after so xcrun FNF is silent
 107         else:
 108             # Use Metal Windows toolkit to test on Linux (Wine) and Windows.
 109             print('Running on non-macOS system.')
 110             subprocess.check_call(['metal', '-x', 'metal', '--version'])
 111
 112     except OSError as e:
 113         if (e.errno != errno.ENOENT):    # Ignore xcrun not found error
 114             raise
 115         print('Metal SDK is not present.\n')
 116     except subprocess.CalledProcessError:
 117         pass
 118
 119 def path_to_msl_standard(shader):
 120     if '.msl31.' in shader:
 121         return '-std=metal3.1'
 122     elif '.msl3.' in shader:
 123         return '-std=metal3.0'
 124     elif '.ios.' in shader:
 125         if '.msl2.' in shader:
 126             return '-std=ios-metal2.0'
 127         elif '.msl21.' in shader:
 128             return '-std=ios-metal2.1'
 129         elif '.msl22.' in shader:
 130             return '-std=ios-metal2.2'
 131         elif '.msl23.' in shader:
 132             return '-std=ios-metal2.3'
 133         elif '.msl24.' in shader:
 134             return '-std=ios-metal2.4'
 135         elif '.msl11.' in shader:
 136             return '-std=ios-metal1.1'
 137         elif '.msl10.' in shader:
 138             return '-std=ios-metal1.0'
 139         else:
 140             return '-std=ios-metal1.2'
 141     else:
 142         if '.msl2.' in shader:
 143             return '-std=macos-metal2.0'
 144         elif '.msl21.' in shader:
 145             return '-std=macos-metal2.1'
 146         elif '.msl22.' in shader:
 147             return '-std=macos-metal2.2'
 148         elif '.msl23.' in shader:
 149             return '-std=macos-metal2.3'
 150         elif '.msl24.' in shader:
 151             return '-std=macos-metal2.4'
 152         elif '.msl11.' in shader:
 153             return '-std=macos-metal1.1'
 154         else:
 155             return '-std=macos-metal1.2'
 156
 157 def path_to_msl_standard_cli(shader):
 158     if '.msl31.' in shader:
 159         return '30100'
 160     elif '.msl3.' in shader:
 161         return '30000'
 162     elif '.msl2.' in shader:
 163         return '20000'
 164     elif '.msl21.' in shader:
 165         return '20100'
 166     elif '.msl22.' in shader:
 167         return '20200'
 168     elif '.msl23.' in shader:
 169         return '20300'
 170     elif '.msl24.' in shader:
 171         return '20400'
 172     elif '.msl11.' in shader:
 173         return '10100'
 174     else:
 175         return '10200'
 176
 177 ignore_win_metal_tool = False
 178 def validate_shader_msl(shader, opt):
 179     msl_path = reference_path(shader[0], shader[1], opt)
 180     global ignore_win_metal_tool
 181     try:
 182         if '.ios.' in msl_path:
 183             msl_os = 'iphoneos'
 184         else:
 185             msl_os = 'macosx'
 186
 187         if platform.system() == 'Darwin':
 188             subprocess.check_call(['xcrun', '--sdk', msl_os, 'metal', '-x', 'metal', path_to_msl_standard(msl_path), '-Werror', '-Wno-unused-variable', msl_path])
 189             print('Compiled Metal shader: ' + msl_path)   # display after so xcrun FNF is silent
 190         elif not ignore_win_metal_tool:
 191             # Use Metal Windows toolkit to test on Linux (Wine) and Windows. Running offline tool on Linux gets weird.
 192             # Normal winepath doesn't work, it must be Z:/abspath *exactly* for some bizarre reason.
 193             target_path = msl_path if platform.system == 'Windows' else ('Z:' + os.path.abspath(msl_path))
 194             subprocess.check_call(['metal', '-x', 'metal', path_to_msl_standard(msl_path), '-Werror', '-Wno-unused-variable', target_path])
 195
 196     except OSError as oe:
 197         if (oe.errno != errno.ENOENT):   # Ignore xcrun or metal not found error
 198             raise
 199         print('metal toolkit does not exist, ignoring further attempts to use it.')
 200         ignore_win_metal_tool = True
 201     except subprocess.CalledProcessError:
 202         print('Error compiling Metal shader: ' + msl_path)
 203         raise RuntimeError('Failed to compile Metal shader')
 204
 205 def cross_compile_msl(shader, spirv, opt, iterations, paths):
 206     spirv_path = create_temporary()
 207     msl_path = create_temporary(os.path.basename(shader))
 208
 209     spirv_16 = '.spv16.' in shader
 210     spirv_14 = '.spv14.' in shader
 211
 212     if spirv_16:
 213         spirv_env = 'spv1.6'
 214         glslang_env = 'spirv1.6'
 215     elif spirv_14:
 216         spirv_env = 'vulkan1.1spv1.4'
 217         glslang_env = 'spirv1.4'
 218     else:
 219         spirv_env = 'vulkan1.1'
 220         glslang_env = 'vulkan1.1'
 221
 222     spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
 223
 224     if spirv:
 225         subprocess.check_call(spirv_cmd)
 226     else:
 227         subprocess.check_call([paths.glslang, '--amb' ,'--target-env', glslang_env, '-V', '-o', spirv_path, shader])
 228
 229     if opt and (not shader_is_invalid_spirv(shader)):
 230         if '.graphics-robust-access.' in shader:
 231             subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '--graphics-robust-access', '-o', spirv_path, spirv_path])
 232         else:
 233             subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
 234
 235     spirv_cross_path = paths.spirv_cross
 236
 237     msl_args = [spirv_cross_path, '--output', msl_path, spirv_path, '--msl', '--iterations', str(iterations)]
 238     msl_args.append('--msl-version')
 239     msl_args.append(path_to_msl_standard_cli(shader))
 240     if not '.nomain.' in shader:
 241         msl_args.append('--entry')
 242         msl_args.append('main')
 243     if '.swizzle.' in shader:
 244         msl_args.append('--msl-swizzle-texture-samples')
 245     if '.ios.' in shader:
 246         msl_args.append('--msl-ios')
 247     if '.pad-fragment.' in shader:
 248         msl_args.append('--msl-pad-fragment-output')
 249     if '.capture.' in shader:
 250         msl_args.append('--msl-capture-output')
 251     if '.domain.' in shader:
 252         msl_args.append('--msl-domain-lower-left')
 253     if '.argument.' in shader:
 254         msl_args.append('--msl-argument-buffers')
 255     if '.argument-tier-1.' in shader:
 256         msl_args.append('--msl-argument-buffer-tier')
 257         msl_args.append('1')
 258     if '.texture-buffer-native.' in shader:
 259         msl_args.append('--msl-texture-buffer-native')
 260     if '.framebuffer-fetch.' in shader:
 261         msl_args.append('--msl-framebuffer-fetch')
 262     if '.invariant-float-math.' in shader:
 263         msl_args.append('--msl-invariant-float-math')
 264     if '.emulate-cube-array.' in shader:
 265         msl_args.append('--msl-emulate-cube-array')
 266     if '.discrete.' in shader:
 267         # Arbitrary for testing purposes.
 268         msl_args.append('--msl-discrete-descriptor-set')
 269         msl_args.append('2')
 270         msl_args.append('--msl-discrete-descriptor-set')
 271         msl_args.append('3')
 272     if '.force-active.' in shader:
 273         msl_args.append('--msl-force-active-argument-buffer-resources')
 274     if '.line.' in shader:
 275         msl_args.append('--emit-line-directives')
 276     if '.multiview.' in shader:
 277         msl_args.append('--msl-multiview')
 278     if '.no-layered.' in shader:
 279         msl_args.append('--msl-multiview-no-layered-rendering')
 280     if '.viewfromdev.' in shader:
 281         msl_args.append('--msl-view-index-from-device-index')
 282     if '.dispatchbase.' in shader:
 283         msl_args.append('--msl-dispatch-base')
 284     if '.dynamic-buffer.' in shader:
 285         # Arbitrary for testing purposes.
 286         msl_args.append('--msl-dynamic-buffer')
 287         msl_args.append('0')
 288         msl_args.append('0')
 289         msl_args.append('--msl-dynamic-buffer')
 290         msl_args.append('1')
 291         msl_args.append('2')
 292     if '.inline-block.' in shader:
 293         # Arbitrary for testing purposes.
 294         msl_args.append('--msl-inline-uniform-block')
 295         msl_args.append('0')
 296         msl_args.append('0')
 297     if '.device-argument-buffer.' in shader:
 298         msl_args.append('--msl-device-argument-buffer')
 299         msl_args.append('0')
 300         msl_args.append('--msl-device-argument-buffer')
 301         msl_args.append('1')
 302     if '.force-native-array.' in shader:
 303         msl_args.append('--msl-force-native-arrays')
 304     if '.zero-initialize.' in shader:
 305         msl_args.append('--force-zero-initialized-variables')
 306     if '.frag-output.' in shader:
 307         # Arbitrary for testing purposes.
 308         msl_args.append('--msl-disable-frag-depth-builtin')
 309         msl_args.append('--msl-disable-frag-stencil-ref-builtin')
 310         msl_args.append('--msl-enable-frag-output-mask')
 311         msl_args.append('0x000000ca')
 312     if '.no-user-varying.' in shader:
 313         msl_args.append('--msl-no-clip-distance-user-varying')
 314     if '.shader-inputs.' in shader:
 315         # Arbitrary for testing purposes.
 316         msl_args.append('--msl-shader-input')
 317         msl_args.append('0')
 318         msl_args.append('u8')
 319         msl_args.append('2')
 320         msl_args.append('--msl-shader-input')
 321         msl_args.append('1')
 322         msl_args.append('u16')
 323         msl_args.append('3')
 324         msl_args.append('--msl-shader-input')
 325         msl_args.append('6')
 326         msl_args.append('other')
 327         msl_args.append('4')
 328     if '.multi-patch.' in shader:
 329         msl_args.append('--msl-multi-patch-workgroup')
 330         # Arbitrary for testing purposes.
 331         msl_args.append('--msl-shader-input')
 332         msl_args.append('0')
 333         msl_args.append('any32')
 334         msl_args.append('3')
 335         msl_args.append('--msl-shader-input')
 336         msl_args.append('1')
 337         msl_args.append('any16')
 338         msl_args.append('2')
 339     if '.raw-tess-in.' in shader:
 340         msl_args.append('--msl-raw-buffer-tese-input')
 341     if '.for-tess.' in shader:
 342         msl_args.append('--msl-vertex-for-tessellation')
 343     if '.fixed-sample-mask.' in shader:
 344         msl_args.append('--msl-additional-fixed-sample-mask')
 345         msl_args.append('0x00000022')
 346     if '.arrayed-subpass.' in shader:
 347         msl_args.append('--msl-arrayed-subpass-input')
 348     if '.1d-as-2d.' in shader:
 349         msl_args.append('--msl-texture-1d-as-2d')
 350     if '.simd.' in shader:
 351         msl_args.append('--msl-ios-use-simdgroup-functions')
 352     if '.emulate-subgroup.' in shader:
 353         msl_args.append('--msl-emulate-subgroups')
 354     if '.fixed-subgroup.' in shader:
 355         # Arbitrary for testing purposes.
 356         msl_args.append('--msl-fixed-subgroup-size')
 357         msl_args.append('32')
 358     if '.force-sample.' in shader:
 359         msl_args.append('--msl-force-sample-rate-shading')
 360     if '.discard-checks.' in shader:
 361         msl_args.append('--msl-check-discarded-frag-stores')
 362     if '.force-frag-with-side-effects-execution.' in shader:
 363         msl_args.append('--msl-force-frag-with-side-effects-execution')
 364     if '.lod-as-grad.' in shader:
 365         msl_args.append('--msl-sample-dref-lod-array-as-grad')
 366     if '.agx-cube-grad.' in shader:
 367         msl_args.append('--msl-agx-manual-cube-grad-fixup')
 368     if '.decoration-binding.' in shader:
 369         msl_args.append('--msl-decoration-binding')
 370     if '.rich-descriptor.' in shader:
 371         msl_args.append('--msl-runtime-array-rich-descriptor')
 372     if '.replace-recursive-inputs.' in shader:
 373         msl_args.append('--msl-replace-recursive-inputs')
 374     if '.input-attachment-is-ds-attachment.' in shader:
 375         msl_args.append('--msl-input-attachment-is-ds-attachment')
 376     if '.mask-location-0.' in shader:
 377         msl_args.append('--mask-stage-output-location')
 378         msl_args.append('0')
 379         msl_args.append('0')
 380     if '.mask-location-1.' in shader:
 381         msl_args.append('--mask-stage-output-location')
 382         msl_args.append('1')
 383         msl_args.append('0')
 384     if '.mask-position.' in shader:
 385         msl_args.append('--mask-stage-output-builtin')
 386         msl_args.append('Position')
 387     if '.mask-point-size.' in shader:
 388         msl_args.append('--mask-stage-output-builtin')
 389         msl_args.append('PointSize')
 390     if '.mask-clip-distance.' in shader:
 391         msl_args.append('--mask-stage-output-builtin')
 392         msl_args.append('ClipDistance')
 393     if '.relax-nan.' in shader:
 394         msl_args.append('--relax-nan-checks')
 395
 396     subprocess.check_call(msl_args)
 397
 398     if not shader_is_invalid_spirv(msl_path):
 399         subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
 400
 401     return (spirv_path, msl_path)
 402
 403 def shader_model_hlsl(shader):
 404     if '.vert' in shader:
 405         if '.sm30.' in shader:
 406             return '-Tvs_3_0'
 407         else:
 408             return '-Tvs_5_1'
 409     elif '.frag' in shader:
 410         if '.sm30.' in shader:
 411             return '-Tps_3_0'
 412         else:
 413             return '-Tps_5_1'
 414     elif '.comp' in shader:
 415         return '-Tcs_5_1'
 416     elif '.mesh' in shader:
 417         return '-Tms_6_5'
 418     elif '.task' in shader:
 419         return '-Tas_6_5'
 420     else:
 421         return None
 422
 423 def shader_to_win_path(shader):
 424     # It's (very) convenient to be able to run HLSL testing in wine on Unix-likes, so support that.
 425     try:
 426         with subprocess.Popen(['winepath', '-w', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE) as f:
 427             stdout_data, stderr_data = f.communicate()
 428             return stdout_data.decode('utf-8')
 429     except OSError as oe:
 430         if (oe.errno != errno.ENOENT): # Ignore not found errors
 431             return shader
 432     except subprocess.CalledProcessError:
 433         raise
 434
 435     return shader
 436
 437 ignore_fxc = False
 438 def validate_shader_hlsl(shader, force_no_external_validation, paths):
 439     test_glslang = True
 440     if '.nonuniformresource.' in shader:
 441         test_glslang = False
 442     if '.fxconly.' in shader:
 443         test_glslang = False
 444     if '.task' in shader or '.mesh' in shader:
 445         test_glslang = False
 446
 447     hlsl_args = [paths.glslang, '--amb', '-e', 'main', '-D', '--target-env', 'vulkan1.1', '-V', shader]
 448     if '.sm30.' in shader:
 449         hlsl_args.append('--hlsl-dx9-compatible')
 450
 451     if test_glslang:
 452         subprocess.check_call(hlsl_args)
 453
 454     is_no_fxc = '.nofxc.' in shader
 455     global ignore_fxc
 456     if (not ignore_fxc) and (not force_no_external_validation) and (not is_no_fxc):
 457         try:
 458             win_path = shader_to_win_path(shader)
 459             args = ['fxc', '-nologo', shader_model_hlsl(shader), win_path]
 460             if '.nonuniformresource.' in shader:
 461                 args.append('/enable_unbounded_descriptor_tables')
 462             subprocess.check_call(args)
 463         except OSError as oe:
 464             if (oe.errno != errno.ENOENT): # Ignore not found errors
 465                 print('Failed to run FXC.')
 466                 ignore_fxc = True
 467                 raise
 468             else:
 469                 print('Could not find FXC.')
 470                 ignore_fxc = True
 471         except subprocess.CalledProcessError:
 472             print('Failed compiling HLSL shader:', shader, 'with FXC.')
 473             raise RuntimeError('Failed compiling HLSL shader')
 474
 475 def shader_to_sm(shader):
 476     if '.sm62.' in shader:
 477         return '62'
 478     elif '.sm61.' in shader:
 479         return '61'
 480     elif '.sm60.' in shader:
 481         return '60'
 482     elif '.sm68.' in shader:
 483         return '68'
 484     elif '.sm51.' in shader:
 485         return '51'
 486     elif '.sm30.' in shader:
 487         return '30'
 488     else:
 489         return '50'
 490
 491 def cross_compile_hlsl(shader, spirv, opt, force_no_external_validation, iterations, paths):
 492     spirv_path = create_temporary()
 493     hlsl_path = create_temporary(os.path.basename(shader))
 494
 495     spirv_16 = '.spv16.' in shader
 496     spirv_14 = '.spv14.' in shader
 497
 498     if spirv_16:
 499         spirv_env = 'spv1.6'
 500         glslang_env = 'spirv1.6'
 501     elif spirv_14:
 502         spirv_env = 'vulkan1.1spv1.4'
 503         glslang_env = 'spirv1.4'
 504     else:
 505         spirv_env = 'vulkan1.1'
 506         glslang_env = 'vulkan1.1'
 507
 508     spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
 509
 510     if spirv:
 511         subprocess.check_call(spirv_cmd)
 512     else:
 513         subprocess.check_call([paths.glslang, '--amb', '--target-env', glslang_env, '-V', '-o', spirv_path, shader])
 514
 515     if opt and (not shader_is_invalid_spirv(hlsl_path)):
 516         subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
 517
 518     spirv_cross_path = paths.spirv_cross
 519
 520     sm = shader_to_sm(shader)
 521
 522     hlsl_args = [spirv_cross_path, '--entry', 'main', '--output', hlsl_path, spirv_path, '--hlsl-enable-compat', '--hlsl', '--shader-model', sm, '--iterations', str(iterations)]
 523     if '.line.' in shader:
 524         hlsl_args.append('--emit-line-directives')
 525     if '.flatten.' in shader:
 526         hlsl_args.append('--flatten-ubo')
 527     if '.force-uav.' in shader:
 528         hlsl_args.append('--hlsl-force-storage-buffer-as-uav')
 529     if '.zero-initialize.' in shader:
 530         hlsl_args.append('--force-zero-initialized-variables')
 531     if '.nonwritable-uav-texture.' in shader:
 532         hlsl_args.append('--hlsl-nonwritable-uav-texture-as-srv')
 533     if '.native-16bit.' in shader:
 534         hlsl_args.append('--hlsl-enable-16bit-types')
 535     if '.flatten-matrix-vertex-input.' in shader:
 536         hlsl_args.append('--hlsl-flatten-matrix-vertex-input-semantics')
 537     if '.relax-nan.' in shader:
 538         hlsl_args.append('--relax-nan-checks')
 539     if '.structured.' in shader:
 540         hlsl_args.append('--hlsl-preserve-structured-buffers')
 541     if '.flip-vert-y.' in shader:
 542         hlsl_args.append('--flip-vert-y')
 543
 544     subprocess.check_call(hlsl_args)
 545
 546     if not shader_is_invalid_spirv(hlsl_path):
 547         subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
 548
 549     validate_shader_hlsl(hlsl_path, force_no_external_validation, paths)
 550
 551     return (spirv_path, hlsl_path)
 552
 553 def cross_compile_reflect(shader, spirv, opt, iterations, paths):
 554     spirv_path = create_temporary()
 555     reflect_path = create_temporary(os.path.basename(shader))
 556
 557     spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', 'vulkan1.1', '-o', spirv_path, shader]
 558
 559     if spirv:
 560         subprocess.check_call(spirv_cmd)
 561     else:
 562         subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader])
 563
 564     if opt and (not shader_is_invalid_spirv(reflect_path)):
 565         subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
 566
 567     spirv_cross_path = paths.spirv_cross
 568
 569     sm = shader_to_sm(shader)
 570     subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', reflect_path, spirv_path, '--reflect', '--iterations', str(iterations)])
 571     return (spirv_path, reflect_path)
 572
 573 def validate_shader(shader, vulkan, paths):
 574     if vulkan:
 575         spirv_14 = '.spv14.' in shader
 576         glslang_env = 'spirv1.4' if spirv_14 else 'vulkan1.1'
 577         subprocess.check_call([paths.glslang, '--amb', '--target-env', glslang_env, '-V', shader])
 578     else:
 579         subprocess.check_call([paths.glslang, shader])
 580
 581 def cross_compile(shader, vulkan, spirv, invalid_spirv, eliminate, is_legacy, force_es, flatten_ubo, sso, flatten_dim, opt, push_ubo, iterations, paths):
 582     spirv_path = create_temporary()
 583     glsl_path = create_temporary(os.path.basename(shader))
 584
 585     spirv_16 = '.spv16.' in shader
 586     spirv_14 = '.spv14.' in shader
 587     if spirv_16:
 588         spirv_env = 'spv1.6'
 589         glslang_env = 'spirv1.6'
 590     elif spirv_14:
 591         spirv_env = 'vulkan1.1spv1.4'
 592         glslang_env = 'spirv1.4'
 593     else:
 594         spirv_env = 'vulkan1.1'
 595         glslang_env = 'vulkan1.1'
 596
 597     if vulkan or spirv:
 598         vulkan_glsl_path = create_temporary('vk' + os.path.basename(shader))
 599
 600     spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
 601
 602     if spirv:
 603         subprocess.check_call(spirv_cmd)
 604     else:
 605         glslang_cmd = [paths.glslang, '--amb', '--target-env', glslang_env, '-V', '-o', spirv_path, shader]
 606         if '.g.' in shader:
 607             glslang_cmd.append('-g')
 608         if '.gV.' in shader:
 609             glslang_cmd.append('-gV')
 610         subprocess.check_call(glslang_cmd)
 611
 612     if opt and (not invalid_spirv):
 613         subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
 614
 615     if not invalid_spirv:
 616         subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
 617
 618     extra_args = ['--iterations', str(iterations)]
 619     if eliminate:
 620         extra_args += ['--remove-unused-variables']
 621     if is_legacy:
 622         extra_args += ['--version', '100', '--es']
 623     if force_es:
 624         extra_args += ['--version', '310', '--es']
 625     if flatten_ubo:
 626         extra_args += ['--flatten-ubo']
 627     if sso:
 628         extra_args += ['--separate-shader-objects']
 629     if flatten_dim:
 630         extra_args += ['--flatten-multidimensional-arrays']
 631     if push_ubo:
 632         extra_args += ['--glsl-emit-push-constant-as-ubo']
 633     if '.line.' in shader:
 634         extra_args += ['--emit-line-directives']
 635     if '.no-samplerless.' in shader:
 636         extra_args += ['--vulkan-glsl-disable-ext-samplerless-texture-functions']
 637     if '.no-qualifier-deduction.' in shader:
 638         extra_args += ['--disable-storage-image-qualifier-deduction']
 639     if '.framebuffer-fetch.' in shader:
 640         extra_args += ['--glsl-remap-ext-framebuffer-fetch', '0', '0']
 641         extra_args += ['--glsl-remap-ext-framebuffer-fetch', '1', '1']
 642         extra_args += ['--glsl-remap-ext-framebuffer-fetch', '2', '2']
 643         extra_args += ['--glsl-remap-ext-framebuffer-fetch', '3', '3']
 644     if '.framebuffer-fetch-noncoherent.' in shader:
 645         extra_args += ['--glsl-ext-framebuffer-fetch-noncoherent']
 646     if '.zero-initialize.' in shader:
 647         extra_args += ['--force-zero-initialized-variables']
 648     if '.force-flattened-io.' in shader:
 649         extra_args += ['--glsl-force-flattened-io-blocks']
 650     if '.relax-nan.' in shader:
 651         extra_args.append('--relax-nan-checks')
 652
 653     spirv_cross_path = paths.spirv_cross
 654
 655     # A shader might not be possible to make valid GLSL from, skip validation for this case.
 656     if (not ('nocompat' in glsl_path)) or (not vulkan):
 657         subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', glsl_path, spirv_path] + extra_args)
 658         if not 'nocompat' in glsl_path:
 659             validate_shader(glsl_path, False, paths)
 660     else:
 661         remove_file(glsl_path)
 662         glsl_path = None
 663
 664     if (vulkan or spirv) and (not is_legacy):
 665         subprocess.check_call([spirv_cross_path, '--entry', 'main', '-V', '--output', vulkan_glsl_path, spirv_path] + extra_args)
 666         validate_shader(vulkan_glsl_path, True, paths)
 667         # SPIR-V shaders might just want to validate Vulkan GLSL output, we don't always care about the output.
 668         if not vulkan:
 669             remove_file(vulkan_glsl_path)
 670
 671     return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None)
 672
 673 def make_unix_newline(buf):
 674     decoded = codecs.decode(buf, 'utf-8')
 675     decoded = decoded.replace('\r', '')
 676     return codecs.encode(decoded, 'utf-8')
 677
 678 def md5_for_file(path):
 679     md5 = hashlib.md5()
 680     with open(path, 'rb') as f:
 681         for chunk in iter(lambda: make_unix_newline(f.read(8192)), b''):
 682             md5.update(chunk)
 683     return md5.digest()
 684
 685 def make_reference_dir(path):
 686     base = os.path.dirname(path)
 687     if not os.path.exists(base):
 688         os.makedirs(base)
 689
 690 def reference_path(directory, relpath, opt):
 691     split_paths = os.path.split(directory)
 692     reference_dir = os.path.join(split_paths[0], 'reference/' + ('opt/' if opt else ''))
 693     reference_dir = os.path.join(reference_dir, split_paths[1])
 694     return os.path.join(reference_dir, relpath)
 695
 696 def regression_check_reflect(shader, json_file, args):
 697     reference = reference_path(shader[0], shader[1], args.opt) + '.json'
 698     joined_path = os.path.join(shader[0], shader[1])
 699     print('Reference shader reflection path:', reference)
 700     if os.path.exists(reference):
 701         actual = md5_for_file(json_file)
 702         expected = md5_for_file(reference)
 703         if actual != expected:
 704             if args.update:
 705                 print('Generated reflection json has changed for {}!'.format(reference))
 706                 # If we expect changes, update the reference file.
 707                 if os.path.exists(reference):
 708                     remove_file(reference)
 709                 make_reference_dir(reference)
 710                 shutil.move(json_file, reference)
 711             else:
 712                 print('Generated reflection json in {} does not match reference {}!'.format(json_file, reference))
 713                 if args.diff:
 714                     diff_path = generate_diff_file(reference, glsl)
 715                     with open(diff_path, 'r') as f:
 716                         print('')
 717                         print('Diff:')
 718                         print(f.read())
 719                         print('')
 720                     remove_file(diff_path)
 721                 else:
 722                     with open(json_file, 'r') as f:
 723                         print('')
 724                         print('Generated:')
 725                         print('======================')
 726                         print(f.read())
 727                         print('======================')
 728                         print('')
 729
 730                 # Otherwise, fail the test. Keep the shader file around so we can inspect.
 731                 if not args.keep:
 732                     remove_file(json_file)
 733
 734                 raise RuntimeError('Does not match reference')
 735         else:
 736             remove_file(json_file)
 737     else:
 738         print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference))
 739         make_reference_dir(reference)
 740         shutil.move(json_file, reference)
 741
 742 def generate_diff_file(origin, generated):
 743     diff_destination = create_temporary()
 744     with open(diff_destination, "w") as f:
 745         try:
 746             subprocess.check_call(["diff", origin, generated], stdout=f)
 747         except subprocess.CalledProcessError as e:
 748             # diff returns 1 when the files are different so we can safely
 749             # ignore this case.
 750             if e.returncode != 1:
 751                 raise e
 752
 753     return diff_destination
 754
 755 def regression_check(shader, glsl, args):
 756     reference = reference_path(shader[0], shader[1], args.opt)
 757     joined_path = os.path.join(shader[0], shader[1])
 758     print('Reference shader path:', reference)
 759
 760     if os.path.exists(reference):
 761         if md5_for_file(glsl) != md5_for_file(reference):
 762             if args.update:
 763                 print('Generated source code has changed for {}!'.format(reference))
 764                 # If we expect changes, update the reference file.
 765                 if os.path.exists(reference):
 766                     remove_file(reference)
 767                 make_reference_dir(reference)
 768                 shutil.move(glsl, reference)
 769             else:
 770                 print('Generated source code in {} does not match reference {}!'.format(glsl, reference))
 771                 if args.diff:
 772                     diff_path = generate_diff_file(reference, glsl)
 773                     with open(diff_path, 'r') as f:
 774                         print('')
 775                         print('Diff:')
 776                         print(f.read())
 777                         print('')
 778                     remove_file(diff_path)
 779                 else:
 780                     with open(glsl, 'r') as f:
 781                         print('')
 782                         print('Generated:')
 783                         print('======================')
 784                         print(f.read())
 785                         print('======================')
 786                         print('')
 787
 788                 # Otherwise, fail the test. Keep the shader file around so we can inspect.
 789                 if not args.keep:
 790                     remove_file(glsl)
 791                 raise RuntimeError('Does not match reference')
 792         else:
 793             remove_file(glsl)
 794     else:
 795         print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference))
 796         make_reference_dir(reference)
 797         shutil.move(glsl, reference)
 798
 799 def shader_is_vulkan(shader):
 800     return '.vk.' in shader
 801
 802 def shader_is_desktop(shader):
 803     return '.desktop.' in shader
 804
 805 def shader_is_eliminate_dead_variables(shader):
 806     return '.noeliminate.' not in shader
 807
 808 def shader_is_spirv(shader):
 809     return '.asm.' in shader
 810
 811 def shader_is_invalid_spirv(shader):
 812     return '.invalid.' in shader
 813
 814 def shader_is_legacy(shader):
 815     return '.legacy.' in shader
 816
 817 def shader_is_force_es(shader):
 818     return '.es.' in shader
 819
 820 def shader_is_flatten_ubo(shader):
 821     return '.flatten.' in shader
 822
 823 def shader_is_sso(shader):
 824     return '.sso.' in shader
 825
 826 def shader_is_flatten_dimensions(shader):
 827     return '.flatten_dim.' in shader
 828
 829 def shader_is_noopt(shader):
 830     return '.noopt.' in shader
 831
 832 def shader_is_push_ubo(shader):
 833     return '.push-ubo.' in shader
 834
 835 def test_shader(stats, shader, args, paths):
 836     joined_path = os.path.join(shader[0], shader[1])
 837     vulkan = shader_is_vulkan(shader[1])
 838     desktop = shader_is_desktop(shader[1])
 839     eliminate = shader_is_eliminate_dead_variables(shader[1])
 840     is_spirv = shader_is_spirv(shader[1])
 841     invalid_spirv = shader_is_invalid_spirv(shader[1])
 842     is_legacy = shader_is_legacy(shader[1])
 843     force_es = shader_is_force_es(shader[1])
 844     flatten_ubo = shader_is_flatten_ubo(shader[1])
 845     sso = shader_is_sso(shader[1])
 846     flatten_dim = shader_is_flatten_dimensions(shader[1])
 847     noopt = shader_is_noopt(shader[1])
 848     push_ubo = shader_is_push_ubo(shader[1])
 849
 850     print('Testing shader:', joined_path)
 851     spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan, is_spirv, invalid_spirv, eliminate, is_legacy, force_es, flatten_ubo, sso, flatten_dim, args.opt and (not noopt), push_ubo, args.iterations, paths)
 852
 853     # Only test GLSL stats if we have a shader following GL semantics.
 854     if stats and (not vulkan) and (not is_spirv) and (not desktop):
 855         cross_stats = get_shader_stats(glsl)
 856
 857     if glsl:
 858         regression_check(shader, glsl, args)
 859     if vulkan_glsl:
 860         regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, args)
 861
 862     remove_file(spirv)
 863
 864     if stats and (not vulkan) and (not is_spirv) and (not desktop):
 865         pristine_stats = get_shader_stats(joined_path)
 866
 867         a = []
 868         a.append(shader[1])
 869         for i in pristine_stats:
 870             a.append(str(i))
 871         for i in cross_stats:
 872             a.append(str(i))
 873         print(','.join(a), file = stats)
 874
 875 def test_shader_msl(stats, shader, args, paths):
 876     joined_path = os.path.join(shader[0], shader[1])
 877     print('\nTesting MSL shader:', joined_path)
 878     is_spirv = shader_is_spirv(shader[1])
 879     noopt = shader_is_noopt(shader[1])
 880     spirv, msl = cross_compile_msl(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths)
 881     regression_check(shader, msl, args)
 882
 883     # Uncomment the following line to print the temp SPIR-V file path.
 884     # This temp SPIR-V file is not deleted until after the Metal validation step below.
 885     # If Metal validation fails, the temp SPIR-V file can be copied out and
 886     # used as input to an invocation of spirv-cross to debug from Xcode directly.
 887     # To do so, build spriv-cross using `make DEBUG=1`, then run the spriv-cross
 888     # executable from Xcode using args: `--msl --entry main --output msl_path spirv_path`.
 889 #    print('SPRIV shader: ' + spirv)
 890
 891     skip_validation = '.invalid.' in joined_path
 892     if (not args.force_no_external_validation) and (not skip_validation):
 893         validate_shader_msl(shader, args.opt)
 894
 895     remove_file(spirv)
 896
 897 def test_shader_hlsl(stats, shader, args, paths):
 898     joined_path = os.path.join(shader[0], shader[1])
 899     print('Testing HLSL shader:', joined_path)
 900     is_spirv = shader_is_spirv(shader[1])
 901     noopt = shader_is_noopt(shader[1])
 902     spirv, hlsl = cross_compile_hlsl(joined_path, is_spirv, args.opt and (not noopt), args.force_no_external_validation, args.iterations, paths)
 903     regression_check(shader, hlsl, args)
 904     remove_file(spirv)
 905
 906 def test_shader_reflect(stats, shader, args, paths):
 907     joined_path = os.path.join(shader[0], shader[1])
 908     print('Testing shader reflection:', joined_path)
 909     is_spirv = shader_is_spirv(shader[1])
 910     noopt = shader_is_noopt(shader[1])
 911     spirv, reflect = cross_compile_reflect(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths)
 912     regression_check_reflect(shader, reflect, args)
 913     remove_file(spirv)
 914
 915 def test_shader_file(relpath, stats, args, backend):
 916     paths = Paths(args.spirv_cross, args.glslang, args.spirv_as, args.spirv_val, args.spirv_opt)
 917     try:
 918         if backend == 'msl':
 919             test_shader_msl(stats, (args.folder, relpath), args, paths)
 920         elif backend == 'hlsl':
 921             test_shader_hlsl(stats, (args.folder, relpath), args, paths)
 922         elif backend == 'reflect':
 923             test_shader_reflect(stats, (args.folder, relpath), args, paths)
 924         else:
 925             test_shader(stats, (args.folder, relpath), args, paths)
 926         return None
 927     except Exception as e:
 928         return e
 929
 930 def test_shaders_helper(stats, backend, args):
 931     all_files = []
 932     for root, dirs, files in os.walk(os.path.join(args.folder)):
 933         files = [ f for f in files if not f.startswith(".") ]   #ignore system files (esp OSX)
 934         for i in files:
 935             path = os.path.join(root, i)
 936             relpath = os.path.relpath(path, args.folder)
 937             all_files.append(relpath)
 938
 939     # The child processes in parallel execution mode don't have the proper state for the global args variable, so
 940     # at this point we need to switch to explicit arguments
 941     if args.parallel:
 942         with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
 943             results = []
 944             for f in all_files:
 945                 results.append(pool.apply_async(test_shader_file,
 946                     args = (f, stats, args, backend)))
 947
 948             pool.close()
 949             pool.join()
 950             results_completed = [res.get() for res in results]
 951
 952             for error in results_completed:
 953                 if error is not None:
 954                     print('Error:', error)
 955                     sys.exit(1)
 956
 957     else:
 958         for i in all_files:
 959             e = test_shader_file(i, stats, args, backend)
 960             if e is not None:
 961                 print('Error:', e)
 962                 sys.exit(1)
 963
 964 def test_shaders(backend, args):
 965     if args.malisc:
 966         with open('stats.csv', 'w') as stats:
 967             print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats)
 968             test_shaders_helper(stats, backend, args)
 969     else:
 970         test_shaders_helper(None, backend, args)
 971
 972 def main():
 973     parser = argparse.ArgumentParser(description = 'Script for regression testing.')
 974     parser.add_argument('folder',
 975             help = 'Folder containing shader files to test.')
 976     parser.add_argument('--update',
 977             action = 'store_true',
 978             help = 'Updates reference files if there is a mismatch. Use when legitimate changes in output is found.')
 979     parser.add_argument('--keep',
 980             action = 'store_true',
 981             help = 'Leave failed GLSL shaders on disk if they fail regression. Useful for debugging.')
 982     parser.add_argument('--diff',
 983             action = 'store_true',
 984             help = 'Displays a diff instead of the generated output on failure. Useful for debugging.')
 985     parser.add_argument('--malisc',
 986             action = 'store_true',
 987             help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.')
 988     parser.add_argument('--msl',
 989             action = 'store_true',
 990             help = 'Test Metal backend.')
 991     parser.add_argument('--metal',
 992             action = 'store_true',
 993             help = 'Deprecated Metal option. Use --msl instead.')
 994     parser.add_argument('--hlsl',
 995             action = 'store_true',
 996             help = 'Test HLSL backend.')
 997     parser.add_argument('--force-no-external-validation',
 998             action = 'store_true',
 999             help = 'Disable all external validation.')
1000     parser.add_argument('--opt',
1001             action = 'store_true',
1002             help = 'Run SPIRV-Tools optimization passes as well.')
1003     parser.add_argument('--reflect',
1004             action = 'store_true',
1005             help = 'Test reflection backend.')
1006     parser.add_argument('--parallel',
1007             action = 'store_true',
1008             help = 'Execute tests in parallel.  Useful for doing regression quickly, but bad for debugging and stat output.')
1009     parser.add_argument('--spirv-cross',
1010             default = './spirv-cross',
1011             help = 'Explicit path to spirv-cross')
1012     parser.add_argument('--glslang',
1013             default = 'glslangValidator',
1014             help = 'Explicit path to glslangValidator')
1015     parser.add_argument('--spirv-as',
1016             default = 'spirv-as',
1017             help = 'Explicit path to spirv-as')
1018     parser.add_argument('--spirv-val',
1019             default = 'spirv-val',
1020             help = 'Explicit path to spirv-val')
1021     parser.add_argument('--spirv-opt',
1022             default = 'spirv-opt',
1023             help = 'Explicit path to spirv-opt')
1024     parser.add_argument('--iterations',
1025             default = 1,
1026             type = int,
1027             help = 'Number of iterations to run SPIRV-Cross (benchmarking)')
1028
1029     args = parser.parse_args()
1030     if not args.folder:
1031         sys.stderr.write('Need shader folder.\n')
1032         sys.exit(1)
1033
1034     if (args.parallel and (args.malisc or args.force_no_external_validation or args.update)):
1035         sys.stderr.write('Parallel execution is disabled when using the flags --update, --malisc or --force-no-external-validation\n')
1036         args.parallel = False
1037
1038     if args.msl:
1039         print_msl_compiler_version()
1040
1041     backend = 'glsl'
1042     if (args.msl or args.metal):
1043         backend = 'msl'
1044     elif args.hlsl:
1045         backend = 'hlsl'
1046     elif args.reflect:
1047         backend = 'reflect'
1048
1049     test_shaders(backend, args)
1050     if args.malisc:
1051         print('Stats in stats.csv!')
1052     print('Tests completed!')
1053
1054 if __name__ == '__main__':
1055     main()