native_client_sdk/src/build_tools/test_projects.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import os
   8 import subprocess
   9 import sys
  10 import time
  11
  12 import build_projects
  13 import build_version
  14 import buildbot_common
  15 import parse_dsc
  16
  17 from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR
  18
  19 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  20 import getos
  21 platform = getos.GetPlatform()
  22
  23 # TODO(binji): ugly hack -- can I get the browser in a cleaner way?
  24 sys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection'))
  25 import find_chrome
  26 browser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release'])
  27
  28 # Fall back to using CHROME_PATH (same as in common.mk)
  29 if not browser_path:
  30   browser_path = os.environ.get('CHROME_PATH')
  31
  32
  33 pepper_ver = str(int(build_version.ChromeMajorVersion()))
  34 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
  35
  36 browser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools',
  37     'browser_tester', 'browser_tester.py')
  38
  39
  40 ALL_CONFIGS = ['Debug', 'Release']
  41 ALL_TOOLCHAINS = [
  42   'newlib',
  43   'glibc',
  44   'pnacl',
  45   'win',
  46   'linux',
  47   'mac',
  48   'clang-newlib',
  49 ]
  50
  51 # Values you can filter by:
  52 #   name: The name of the test. (e.g. "pi_generator")
  53 #   config: See ALL_CONFIGS above.
  54 #   toolchain: See ALL_TOOLCHAINS above.
  55 #   platform: mac/win/linux.
  56 #
  57 # All keys must be matched, but any value that matches in a sequence is
  58 # considered a match for that key. For example:
  59 #
  60 #   {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')}
  61 #
  62 # Will match 8 tests:
  63 #   pi_generator.newlib_debug_test
  64 #   pi_generator.newlib_release_test
  65 #   input_event.newlib_debug_test
  66 #   input_event.newlib_release_test
  67 #   pi_generator.glibc_debug_test
  68 #   pi_generator.glibc_release_test
  69 #   input_event.glibc_debug_test
  70 #   input_event.glibc_release_test
  71 DISABLED_TESTS = [
  72     # TODO(binji): Disable 3D examples on linux/win/mac. See
  73     # http://crbug.com/262379.
  74     {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')},
  75     {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')},
  76     # media_stream_audio uses audio input devices which are not supported.
  77     {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')},
  78     # media_stream_video uses 3D and webcam which are not supported.
  79     {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')},
  80     # TODO(binji): These tests timeout on the trybots because the NEXEs take
  81     # more than 40 seconds to load (!). See http://crbug.com/280753
  82     {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'},
  83     # We don't test "getting_started/part1" because it would complicate the
  84     # example.
  85     # TODO(binji): figure out a way to inject the testing code without
  86     # modifying the example; maybe an extension?
  87     {'name': 'part1'},
  88 ]
  89
  90 def ValidateToolchains(toolchains):
  91   invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS)
  92   if invalid_toolchains:
  93     buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
  94         ', '.join(invalid_toolchains)))
  95
  96
  97 def GetServingDirForProject(desc):
  98   dest = desc['DEST']
  99   path = os.path.join(pepperdir, *dest.split('/'))
 100   return os.path.join(path, desc['NAME'])
 101
 102
 103 def GetRepoServingDirForProject(desc):
 104   # This differs from GetServingDirForProject, because it returns the location
 105   # within the Chrome repository of the project, not the "pepperdir".
 106   return os.path.dirname(desc['FILEPATH'])
 107
 108
 109 def GetExecutableDirForProject(desc, toolchain, config):
 110   return os.path.join(GetServingDirForProject(desc), toolchain, config)
 111
 112
 113 def GetBrowserTesterCommand(desc, toolchain, config):
 114   if browser_path is None:
 115     buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.')
 116
 117   args = [
 118     sys.executable,
 119     browser_tester_py,
 120     '--browser_path', browser_path,
 121     '--timeout', '30.0',  # seconds
 122     # Prevent the infobar that shows up when requesting filesystem quota.
 123     '--browser_flag', '--unlimited-storage',
 124     '--enable_sockets',
 125     # Prevent installing a new copy of PNaCl.
 126     '--browser_flag', '--disable-component-update',
 127   ]
 128
 129   args.extend(['--serving_dir', GetServingDirForProject(desc)])
 130   # Fall back on the example directory in the Chromium repo, to find test.js.
 131   args.extend(['--serving_dir', GetRepoServingDirForProject(desc)])
 132   # If it is not found there, fall back on the dummy one (in this directory.)
 133   args.extend(['--serving_dir', SCRIPT_DIR])
 134
 135   if toolchain == platform:
 136     exe_dir = GetExecutableDirForProject(desc, toolchain, config)
 137     ppapi_plugin = os.path.join(exe_dir, desc['NAME'])
 138     if platform == 'win':
 139       ppapi_plugin += '.dll'
 140     else:
 141       ppapi_plugin += '.so'
 142     args.extend(['--ppapi_plugin', ppapi_plugin])
 143
 144     ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower()
 145     args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype])
 146
 147   if toolchain == 'pnacl':
 148     args.extend(['--browser_flag', '--enable-pnacl'])
 149
 150   url = 'index.html'
 151   url += '?tc=%s&config=%s&test=true' % (toolchain, config)
 152   args.extend(['--url', url])
 153   return args
 154
 155
 156 def GetBrowserTesterEnv():
 157   # browser_tester imports tools/valgrind/memcheck_analyze, which imports
 158   # tools/valgrind/common. Well, it tries to, anyway, but instead imports
 159   # common from PYTHONPATH first (which on the buildbots, is a
 160   # common/__init__.py file...).
 161   #
 162   # Clear the PYTHONPATH so it imports the correct file.
 163   env = dict(os.environ)
 164   env['PYTHONPATH'] = ''
 165   return env
 166
 167
 168 def RunTestOnce(desc, toolchain, config):
 169   args = GetBrowserTesterCommand(desc, toolchain, config)
 170   env = GetBrowserTesterEnv()
 171   start_time = time.time()
 172   try:
 173     subprocess.check_call(args, env=env)
 174     result = True
 175   except subprocess.CalledProcessError:
 176     result = False
 177   elapsed = (time.time() - start_time) * 1000
 178   return result, elapsed
 179
 180
 181 def RunTestNTimes(desc, toolchain, config, times):
 182   total_elapsed = 0
 183   for _ in xrange(times):
 184     result, elapsed = RunTestOnce(desc, toolchain, config)
 185     total_elapsed += elapsed
 186     if result:
 187       # Success, stop retrying.
 188       break
 189   return result, total_elapsed
 190
 191
 192 def RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times):
 193   test_name = GetTestName(desc, toolchain, config)
 194   WriteGtestHeader(test_name)
 195   result, elapsed = RunTestNTimes(desc, toolchain, config,
 196                                   retry_on_failure_times)
 197   WriteGtestFooter(result, test_name, elapsed)
 198   return result
 199
 200
 201 def WriteGtestHeader(test_name):
 202   print '\n[ RUN      ] %s' % test_name
 203   sys.stdout.flush()
 204   sys.stderr.flush()
 205
 206
 207 def WriteGtestFooter(success, test_name, elapsed):
 208   sys.stdout.flush()
 209   sys.stderr.flush()
 210   if success:
 211     message = '[       OK ]'
 212   else:
 213     message = '[  FAILED  ]'
 214   print '%s %s (%d ms)' % (message, test_name, elapsed)
 215
 216
 217 def GetTestName(desc, toolchain, config):
 218   return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower())
 219
 220
 221 def IsTestDisabled(desc, toolchain, config):
 222   def AsList(value):
 223     if type(value) not in (list, tuple):
 224       return [value]
 225     return value
 226
 227   def TestMatchesDisabled(test_values, disabled_test):
 228     for key in test_values:
 229       if key in disabled_test:
 230         if test_values[key] not in AsList(disabled_test[key]):
 231           return False
 232     return True
 233
 234   test_values = {
 235       'name': desc['NAME'],
 236       'toolchain': toolchain,
 237       'config': config,
 238       'platform': platform
 239   }
 240
 241   for disabled_test in DISABLED_TESTS:
 242     if TestMatchesDisabled(test_values, disabled_test):
 243       return True
 244   return False
 245
 246
 247 def WriteHorizontalBar():
 248   print '-' * 80
 249
 250
 251 def WriteBanner(message):
 252   WriteHorizontalBar()
 253   print message
 254   WriteHorizontalBar()
 255
 256
 257 def RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times):
 258   tests_run = 0
 259   total_tests = 0
 260   failed = []
 261   disabled = []
 262
 263   for _, desc in parse_dsc.GenerateProjects(tree):
 264     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 265     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 266     valid_configs = set(configs) & set(desc_configs)
 267     for toolchain in sorted(valid_toolchains):
 268       for config in sorted(valid_configs):
 269         test_name = GetTestName(desc, toolchain, config)
 270         total_tests += 1
 271         if IsTestDisabled(desc, toolchain, config):
 272           disabled.append(test_name)
 273           continue
 274
 275         tests_run += 1
 276         success = RunTestWithGtestOutput(desc, toolchain, config,
 277                                          retry_on_failure_times)
 278         if not success:
 279           failed.append(test_name)
 280
 281   if failed:
 282     WriteBanner('FAILED TESTS')
 283     for test in failed:
 284       print '  %s failed.' % test
 285
 286   if disabled:
 287     WriteBanner('DISABLED TESTS')
 288     for test in disabled:
 289       print '  %s disabled.' % test
 290
 291   WriteHorizontalBar()
 292   print 'Tests run: %d/%d (%d disabled).' % (
 293       tests_run, total_tests, len(disabled))
 294   print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run)
 295
 296   success = len(failed) != 0
 297   return success
 298
 299
 300 def BuildAllTestsInTree(tree, toolchains, configs):
 301   for branch, desc in parse_dsc.GenerateProjects(tree):
 302     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 303     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 304     valid_configs = set(configs) & set(desc_configs)
 305     for toolchain in sorted(valid_toolchains):
 306       for config in sorted(valid_configs):
 307         name = '%s/%s' % (branch, desc['NAME'])
 308         build_projects.BuildProjectsBranch(pepperdir, name, deps=False,
 309                                            clean=False, config=config,
 310                                            args=['TOOLCHAIN=%s' % toolchain])
 311
 312
 313 def GetProjectTree(include):
 314   # Everything in src is a library, and cannot be run.
 315   exclude = {'DEST': 'src'}
 316   try:
 317     return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include,
 318                                      exclude=exclude)
 319   except parse_dsc.ValidationError as e:
 320     buildbot_common.ErrorExit(str(e))
 321
 322
 323 def main(args):
 324   parser = argparse.ArgumentParser(description=__doc__)
 325   parser.add_argument('-c', '--config',
 326       help='Choose configuration to run (Debug or Release).  Runs both '
 327            'by default', action='append')
 328   parser.add_argument('-x', '--experimental',
 329       help='Run experimental projects', action='store_true')
 330   parser.add_argument('-t', '--toolchain',
 331       help='Run using toolchain. Can be passed more than once.',
 332       action='append', default=[])
 333   parser.add_argument('-d', '--dest',
 334       help='Select which destinations (project types) are valid.',
 335       action='append')
 336   parser.add_argument('-b', '--build',
 337       help='Build each project before testing.', action='store_true')
 338   parser.add_argument('--retry-times',
 339       help='Number of types to retry on failure', type=int, default=1)
 340   parser.add_argument('projects', nargs='*')
 341
 342   options = parser.parse_args(args)
 343
 344   if not options.toolchain:
 345     options.toolchain = ['newlib', 'glibc', 'pnacl', 'host']
 346
 347   if 'host' in options.toolchain:
 348     options.toolchain.remove('host')
 349     options.toolchain.append(platform)
 350     print 'Adding platform: ' + platform
 351
 352   ValidateToolchains(options.toolchain)
 353
 354   include = {}
 355   if options.toolchain:
 356     include['TOOLS'] = options.toolchain
 357     print 'Filter by toolchain: ' + str(options.toolchain)
 358   if not options.experimental:
 359     include['EXPERIMENTAL'] = False
 360   if options.dest:
 361     include['DEST'] = options.dest
 362     print 'Filter by type: ' + str(options.dest)
 363   if options.projects:
 364     include['NAME'] = options.projects
 365     print 'Filter by name: ' + str(options.projects)
 366   if not options.config:
 367     options.config = ALL_CONFIGS
 368
 369   project_tree = GetProjectTree(include)
 370   if options.build:
 371     BuildAllTestsInTree(project_tree, options.toolchain, options.config)
 372
 373   return RunAllTestsInTree(project_tree, options.toolchain, options.config,
 374                            options.retry_times)
 375
 376
 377 if __name__ == '__main__':
 378   script_name = os.path.basename(sys.argv[0])
 379   try:
 380     sys.exit(main(sys.argv[1:]))
 381   except parse_dsc.ValidationError as e:
 382     buildbot_common.ErrorExit('%s: %s' % (script_name, e))
 383   except KeyboardInterrupt:
 384     buildbot_common.ErrorExit('%s: interrupted' % script_name)