native_client_sdk/src/build_tools/test_projects.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import os
   8 import subprocess
   9 import sys
  10 import time
  11
  12 import build_projects
  13 import build_version
  14 import buildbot_common
  15 import parse_dsc
  16
  17 from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR
  18
  19 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  20 import getos
  21 platform = getos.GetPlatform()
  22
  23 # TODO(binji): ugly hack -- can I get the browser in a cleaner way?
  24 sys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection'))
  25 import find_chrome
  26 browser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release'])
  27
  28 # Fall back to using CHROME_PATH (same as in common.mk)
  29 if not browser_path:
  30   browser_path = os.environ.get('CHROME_PATH')
  31
  32
  33 pepper_ver = str(int(build_version.ChromeMajorVersion()))
  34 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
  35
  36 browser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools',
  37     'browser_tester', 'browser_tester.py')
  38
  39
  40 ALL_CONFIGS = ['Debug', 'Release']
  41 ALL_TOOLCHAINS = [
  42   'newlib',
  43   'glibc',
  44   'pnacl',
  45   'win',
  46   'linux',
  47   'mac',
  48   'clang-newlib',
  49 ]
  50
  51 # Values you can filter by:
  52 #   name: The name of the test. (e.g. "pi_generator")
  53 #   config: See ALL_CONFIGS above.
  54 #   toolchain: See ALL_TOOLCHAINS above.
  55 #   platform: mac/win/linux.
  56 #
  57 # All keys must be matched, but any value that matches in a sequence is
  58 # considered a match for that key. For example:
  59 #
  60 #   {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')}
  61 #
  62 # Will match 8 tests:
  63 #   pi_generator.newlib_debug_test
  64 #   pi_generator.newlib_release_test
  65 #   input_event.newlib_debug_test
  66 #   input_event.newlib_release_test
  67 #   pi_generator.glibc_debug_test
  68 #   pi_generator.glibc_release_test
  69 #   input_event.glibc_debug_test
  70 #   input_event.glibc_release_test
  71 DISABLED_TESTS = [
  72     # TODO(binji): Disable 3D examples on linux/win/mac. See
  73     # http://crbug.com/262379.
  74     {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')},
  75     {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')},
  76     {'name': 'video_encode', 'platform': ('win', 'linux', 'mac')},
  77     # TODO(sbc): Disable pi_generator on linux/win/mac. See
  78     # http://crbug.com/475255.
  79     {'name': 'pi_generator', 'platform': ('win', 'linux', 'mac')},
  80     # media_stream_audio uses audio input devices which are not supported.
  81     {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')},
  82     # media_stream_video uses 3D and webcam which are not supported.
  83     {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')},
  84     # TODO(binji): These tests timeout on the trybots because the NEXEs take
  85     # more than 40 seconds to load (!). See http://crbug.com/280753
  86     {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'},
  87     # We don't test "getting_started/part1" because it would complicate the
  88     # example.
  89     # TODO(binji): figure out a way to inject the testing code without
  90     # modifying the example; maybe an extension?
  91     {'name': 'part1'},
  92 ]
  93
  94 def ValidateToolchains(toolchains):
  95   invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS)
  96   if invalid_toolchains:
  97     buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
  98         ', '.join(invalid_toolchains)))
  99
 100
 101 def GetServingDirForProject(desc):
 102   dest = desc['DEST']
 103   path = os.path.join(pepperdir, *dest.split('/'))
 104   return os.path.join(path, desc['NAME'])
 105
 106
 107 def GetRepoServingDirForProject(desc):
 108   # This differs from GetServingDirForProject, because it returns the location
 109   # within the Chrome repository of the project, not the "pepperdir".
 110   return os.path.dirname(desc['FILEPATH'])
 111
 112
 113 def GetExecutableDirForProject(desc, toolchain, config):
 114   return os.path.join(GetServingDirForProject(desc), toolchain, config)
 115
 116
 117 def GetBrowserTesterCommand(desc, toolchain, config):
 118   if browser_path is None:
 119     buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.')
 120
 121   args = [
 122     sys.executable,
 123     browser_tester_py,
 124     '--browser_path', browser_path,
 125     '--timeout', '30.0',  # seconds
 126     # Prevent the infobar that shows up when requesting filesystem quota.
 127     '--browser_flag', '--unlimited-storage',
 128     '--enable_sockets',
 129     # Prevent installing a new copy of PNaCl.
 130     '--browser_flag', '--disable-component-update',
 131   ]
 132
 133   args.extend(['--serving_dir', GetServingDirForProject(desc)])
 134   # Fall back on the example directory in the Chromium repo, to find test.js.
 135   args.extend(['--serving_dir', GetRepoServingDirForProject(desc)])
 136   # If it is not found there, fall back on the dummy one (in this directory.)
 137   args.extend(['--serving_dir', SCRIPT_DIR])
 138
 139   if toolchain == platform:
 140     exe_dir = GetExecutableDirForProject(desc, toolchain, config)
 141     ppapi_plugin = os.path.join(exe_dir, desc['NAME'])
 142     if platform == 'win':
 143       ppapi_plugin += '.dll'
 144     else:
 145       ppapi_plugin += '.so'
 146     args.extend(['--ppapi_plugin', ppapi_plugin])
 147
 148     ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower()
 149     args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype])
 150
 151   if toolchain == 'pnacl':
 152     args.extend(['--browser_flag', '--enable-pnacl'])
 153
 154   url = 'index.html'
 155   url += '?tc=%s&config=%s&test=true' % (toolchain, config)
 156   args.extend(['--url', url])
 157   return args
 158
 159
 160 def GetBrowserTesterEnv():
 161   # browser_tester imports tools/valgrind/memcheck_analyze, which imports
 162   # tools/valgrind/common. Well, it tries to, anyway, but instead imports
 163   # common from PYTHONPATH first (which on the buildbots, is a
 164   # common/__init__.py file...).
 165   #
 166   # Clear the PYTHONPATH so it imports the correct file.
 167   env = dict(os.environ)
 168   env['PYTHONPATH'] = ''
 169   return env
 170
 171
 172 def RunTestOnce(desc, toolchain, config):
 173   args = GetBrowserTesterCommand(desc, toolchain, config)
 174   env = GetBrowserTesterEnv()
 175   start_time = time.time()
 176   try:
 177     subprocess.check_call(args, env=env)
 178     result = True
 179   except subprocess.CalledProcessError:
 180     result = False
 181   elapsed = (time.time() - start_time) * 1000
 182   return result, elapsed
 183
 184
 185 def RunTestNTimes(desc, toolchain, config, times):
 186   total_elapsed = 0
 187   for _ in xrange(times):
 188     result, elapsed = RunTestOnce(desc, toolchain, config)
 189     total_elapsed += elapsed
 190     if result:
 191       # Success, stop retrying.
 192       break
 193   return result, total_elapsed
 194
 195
 196 def RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times):
 197   test_name = GetTestName(desc, toolchain, config)
 198   WriteGtestHeader(test_name)
 199   result, elapsed = RunTestNTimes(desc, toolchain, config,
 200                                   retry_on_failure_times)
 201   WriteGtestFooter(result, test_name, elapsed)
 202   return result
 203
 204
 205 def WriteGtestHeader(test_name):
 206   print '\n[ RUN      ] %s' % test_name
 207   sys.stdout.flush()
 208   sys.stderr.flush()
 209
 210
 211 def WriteGtestFooter(success, test_name, elapsed):
 212   sys.stdout.flush()
 213   sys.stderr.flush()
 214   if success:
 215     message = '[       OK ]'
 216   else:
 217     message = '[  FAILED  ]'
 218   print '%s %s (%d ms)' % (message, test_name, elapsed)
 219
 220
 221 def GetTestName(desc, toolchain, config):
 222   return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower())
 223
 224
 225 def IsTestDisabled(desc, toolchain, config):
 226   def AsList(value):
 227     if type(value) not in (list, tuple):
 228       return [value]
 229     return value
 230
 231   def TestMatchesDisabled(test_values, disabled_test):
 232     for key in test_values:
 233       if key in disabled_test:
 234         if test_values[key] not in AsList(disabled_test[key]):
 235           return False
 236     return True
 237
 238   test_values = {
 239       'name': desc['NAME'],
 240       'toolchain': toolchain,
 241       'config': config,
 242       'platform': platform
 243   }
 244
 245   for disabled_test in DISABLED_TESTS:
 246     if TestMatchesDisabled(test_values, disabled_test):
 247       return True
 248   return False
 249
 250
 251 def WriteHorizontalBar():
 252   print '-' * 80
 253
 254
 255 def WriteBanner(message):
 256   WriteHorizontalBar()
 257   print message
 258   WriteHorizontalBar()
 259
 260
 261 def RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times):
 262   tests_run = 0
 263   total_tests = 0
 264   failed = []
 265   disabled = []
 266
 267   for _, desc in parse_dsc.GenerateProjects(tree):
 268     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 269     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 270     valid_configs = set(configs) & set(desc_configs)
 271     for toolchain in sorted(valid_toolchains):
 272       for config in sorted(valid_configs):
 273         test_name = GetTestName(desc, toolchain, config)
 274         total_tests += 1
 275         if IsTestDisabled(desc, toolchain, config):
 276           disabled.append(test_name)
 277           continue
 278
 279         tests_run += 1
 280         success = RunTestWithGtestOutput(desc, toolchain, config,
 281                                          retry_on_failure_times)
 282         if not success:
 283           failed.append(test_name)
 284
 285   if failed:
 286     WriteBanner('FAILED TESTS')
 287     for test in failed:
 288       print '  %s failed.' % test
 289
 290   if disabled:
 291     WriteBanner('DISABLED TESTS')
 292     for test in disabled:
 293       print '  %s disabled.' % test
 294
 295   WriteHorizontalBar()
 296   print 'Tests run: %d/%d (%d disabled).' % (
 297       tests_run, total_tests, len(disabled))
 298   print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run)
 299
 300   success = len(failed) != 0
 301   return success
 302
 303
 304 def BuildAllTestsInTree(tree, toolchains, configs):
 305   for branch, desc in parse_dsc.GenerateProjects(tree):
 306     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 307     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 308     valid_configs = set(configs) & set(desc_configs)
 309     for toolchain in sorted(valid_toolchains):
 310       for config in sorted(valid_configs):
 311         name = '%s/%s' % (branch, desc['NAME'])
 312         build_projects.BuildProjectsBranch(pepperdir, name, deps=False,
 313                                            clean=False, config=config,
 314                                            args=['TOOLCHAIN=%s' % toolchain])
 315
 316
 317 def GetProjectTree(include):
 318   # Everything in src is a library, and cannot be run.
 319   exclude = {'DEST': 'src'}
 320   try:
 321     return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include,
 322                                      exclude=exclude)
 323   except parse_dsc.ValidationError as e:
 324     buildbot_common.ErrorExit(str(e))
 325
 326
 327 def main(args):
 328   parser = argparse.ArgumentParser(description=__doc__)
 329   parser.add_argument('-c', '--config',
 330       help='Choose configuration to run (Debug or Release).  Runs both '
 331            'by default', action='append')
 332   parser.add_argument('-x', '--experimental',
 333       help='Run experimental projects', action='store_true')
 334   parser.add_argument('-t', '--toolchain',
 335       help='Run using toolchain. Can be passed more than once.',
 336       action='append', default=[])
 337   parser.add_argument('-d', '--dest',
 338       help='Select which destinations (project types) are valid.',
 339       action='append')
 340   parser.add_argument('-b', '--build',
 341       help='Build each project before testing.', action='store_true')
 342   parser.add_argument('--retry-times',
 343       help='Number of types to retry on failure', type=int, default=1)
 344   parser.add_argument('projects', nargs='*')
 345
 346   options = parser.parse_args(args)
 347
 348   if not options.toolchain:
 349     options.toolchain = ['newlib', 'glibc', 'pnacl', 'host']
 350
 351   if 'host' in options.toolchain:
 352     options.toolchain.remove('host')
 353     options.toolchain.append(platform)
 354     print 'Adding platform: ' + platform
 355
 356   ValidateToolchains(options.toolchain)
 357
 358   include = {}
 359   if options.toolchain:
 360     include['TOOLS'] = options.toolchain
 361     print 'Filter by toolchain: ' + str(options.toolchain)
 362   if not options.experimental:
 363     include['EXPERIMENTAL'] = False
 364   if options.dest:
 365     include['DEST'] = options.dest
 366     print 'Filter by type: ' + str(options.dest)
 367   if options.projects:
 368     include['NAME'] = options.projects
 369     print 'Filter by name: ' + str(options.projects)
 370   if not options.config:
 371     options.config = ALL_CONFIGS
 372
 373   project_tree = GetProjectTree(include)
 374   if options.build:
 375     BuildAllTestsInTree(project_tree, options.toolchain, options.config)
 376
 377   return RunAllTestsInTree(project_tree, options.toolchain, options.config,
 378                            options.retry_times)
 379
 380
 381 if __name__ == '__main__':
 382   script_name = os.path.basename(sys.argv[0])
 383   try:
 384     sys.exit(main(sys.argv[1:]))
 385   except parse_dsc.ValidationError as e:
 386     buildbot_common.ErrorExit('%s: %s' % (script_name, e))
 387   except KeyboardInterrupt:
 388     buildbot_common.ErrorExit('%s: interrupted' % script_name)