native_client_sdk/src/build_tools/test_projects.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import optparse
   7 import os
   8 import subprocess
   9 import sys
  10 import time
  11
  12 import build_projects
  13 import build_version
  14 import buildbot_common
  15 import parse_dsc
  16
  17 from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR
  18
  19 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  20 import getos
  21 platform = getos.GetPlatform()
  22
  23 # TODO(binji): ugly hack -- can I get the browser in a cleaner way?
  24 sys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection'))
  25 import find_chrome
  26 browser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release'])
  27
  28
  29 pepper_ver = str(int(build_version.ChromeMajorVersion()))
  30 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
  31
  32 browser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools',
  33     'browser_tester', 'browser_tester.py')
  34
  35
  36 ALL_CONFIGS = ['Debug', 'Release']
  37 ALL_TOOLCHAINS = ['newlib', 'glibc', 'pnacl', 'win', 'linux', 'mac']
  38
  39 # Values you can filter by:
  40 #   name: The name of the test. (e.g. "pi_generator")
  41 #   config: See ALL_CONFIGS above.
  42 #   toolchain: See ALL_TOOLCHAINS above.
  43 #   platform: mac/win/linux.
  44 #
  45 # All keys must be matched, but any value that matches in a sequence is
  46 # considered a match for that key. For example:
  47 #
  48 #   {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')}
  49 #
  50 # Will match 8 tests:
  51 #   pi_generator.newlib_debug_test
  52 #   pi_generator.newlib_release_test
  53 #   input_event.newlib_debug_test
  54 #   input_event.newlib_release_test
  55 #   pi_generator.glibc_debug_test
  56 #   pi_generator.glibc_release_test
  57 #   input_event.glibc_debug_test
  58 #   input_event.glibc_release_test
  59 DISABLED_TESTS = [
  60     # TODO(binji): Disable 3D examples on linux/win/mac. See
  61     # http://crbug.com/262379.
  62     {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')},
  63     {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')},
  64     # media_stream_audio uses audio input devices which are not supported.
  65     {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')},
  66     # media_stream_video uses 3D and webcam which are not supported.
  67     {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')},
  68     # TODO(binji): These tests timeout on the trybots because the NEXEs take
  69     # more than 40 seconds to load (!). See http://crbug.com/280753
  70     {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'},
  71     # We don't test "getting_started/part1" because it would complicate the
  72     # example.
  73     # TODO(binji): figure out a way to inject the testing code without
  74     # modifying the example; maybe an extension?
  75     {'name': 'part1'},
  76 ]
  77
  78 def ValidateToolchains(toolchains):
  79   invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS)
  80   if invalid_toolchains:
  81     buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
  82         ', '.join(invalid_toolchains)))
  83
  84
  85 def GetServingDirForProject(desc):
  86   dest = desc['DEST']
  87   path = os.path.join(pepperdir, *dest.split('/'))
  88   return os.path.join(path, desc['NAME'])
  89
  90
  91 def GetRepoServingDirForProject(desc):
  92   # This differs from GetServingDirForProject, because it returns the location
  93   # within the Chrome repository of the project, not the "pepperdir".
  94   return os.path.dirname(desc['FILEPATH'])
  95
  96
  97 def GetExecutableDirForProject(desc, toolchain, config):
  98   return os.path.join(GetServingDirForProject(desc), toolchain, config)
  99
 100
 101 def GetBrowserTesterCommand(desc, toolchain, config):
 102   if browser_path is None:
 103     buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.')
 104
 105   args = [
 106     sys.executable,
 107     browser_tester_py,
 108     '--browser_path', browser_path,
 109     '--timeout', '30.0',  # seconds
 110     # Prevent the infobar that shows up when requesting filesystem quota.
 111     '--browser_flag', '--unlimited-storage',
 112     '--enable_sockets',
 113     # Prevent installing a new copy of PNaCl.
 114     '--browser_flag', '--disable-component-update',
 115   ]
 116
 117   args.extend(['--serving_dir', GetServingDirForProject(desc)])
 118   # Fall back on the example directory in the Chromium repo, to find test.js.
 119   args.extend(['--serving_dir', GetRepoServingDirForProject(desc)])
 120   # If it is not found there, fall back on the dummy one (in this directory.)
 121   args.extend(['--serving_dir', SCRIPT_DIR])
 122
 123   if toolchain == platform:
 124     exe_dir = GetExecutableDirForProject(desc, toolchain, config)
 125     ppapi_plugin = os.path.join(exe_dir, desc['NAME'])
 126     if platform == 'win':
 127       ppapi_plugin += '.dll'
 128     else:
 129       ppapi_plugin += '.so'
 130     args.extend(['--ppapi_plugin', ppapi_plugin])
 131
 132     ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower()
 133     args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype])
 134
 135   if toolchain == 'pnacl':
 136     args.extend(['--browser_flag', '--enable-pnacl'])
 137
 138   url = 'index.html'
 139   url += '?tc=%s&config=%s&test=true' % (toolchain, config)
 140   args.extend(['--url', url])
 141   return args
 142
 143
 144 def GetBrowserTesterEnv():
 145   # browser_tester imports tools/valgrind/memcheck_analyze, which imports
 146   # tools/valgrind/common. Well, it tries to, anyway, but instead imports
 147   # common from PYTHONPATH first (which on the buildbots, is a
 148   # common/__init__.py file...).
 149   #
 150   # Clear the PYTHONPATH so it imports the correct file.
 151   env = dict(os.environ)
 152   env['PYTHONPATH'] = ''
 153   return env
 154
 155
 156 def RunTestOnce(desc, toolchain, config):
 157   args = GetBrowserTesterCommand(desc, toolchain, config)
 158   env = GetBrowserTesterEnv()
 159   start_time = time.time()
 160   try:
 161     subprocess.check_call(args, env=env)
 162     result = True
 163   except subprocess.CalledProcessError:
 164     result = False
 165   elapsed = (time.time() - start_time) * 1000
 166   return result, elapsed
 167
 168
 169 def RunTestNTimes(desc, toolchain, config, times):
 170   total_elapsed = 0
 171   for _ in xrange(times):
 172     result, elapsed = RunTestOnce(desc, toolchain, config)
 173     total_elapsed += elapsed
 174     if result:
 175       # Success, stop retrying.
 176       break
 177   return result, total_elapsed
 178
 179
 180 def RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times):
 181   test_name = GetTestName(desc, toolchain, config)
 182   WriteGtestHeader(test_name)
 183   result, elapsed = RunTestNTimes(desc, toolchain, config,
 184                                   retry_on_failure_times)
 185   WriteGtestFooter(result, test_name, elapsed)
 186   return result
 187
 188
 189 def WriteGtestHeader(test_name):
 190   print '\n[ RUN      ] %s' % test_name
 191   sys.stdout.flush()
 192   sys.stderr.flush()
 193
 194
 195 def WriteGtestFooter(success, test_name, elapsed):
 196   sys.stdout.flush()
 197   sys.stderr.flush()
 198   if success:
 199     message = '[       OK ]'
 200   else:
 201     message = '[  FAILED  ]'
 202   print '%s %s (%d ms)' % (message, test_name, elapsed)
 203
 204
 205 def GetTestName(desc, toolchain, config):
 206   return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower())
 207
 208
 209 def IsTestDisabled(desc, toolchain, config):
 210   def AsList(value):
 211     if type(value) not in (list, tuple):
 212       return [value]
 213     return value
 214
 215   def TestMatchesDisabled(test_values, disabled_test):
 216     for key in test_values:
 217       if key in disabled_test:
 218         if test_values[key] not in AsList(disabled_test[key]):
 219           return False
 220     return True
 221
 222   test_values = {
 223       'name': desc['NAME'],
 224       'toolchain': toolchain,
 225       'config': config,
 226       'platform': platform
 227   }
 228
 229   for disabled_test in DISABLED_TESTS:
 230     if TestMatchesDisabled(test_values, disabled_test):
 231       return True
 232   return False
 233
 234
 235 def WriteHorizontalBar():
 236   print '-' * 80
 237
 238
 239 def WriteBanner(message):
 240   WriteHorizontalBar()
 241   print message
 242   WriteHorizontalBar()
 243
 244
 245 def RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times):
 246   tests_run = 0
 247   total_tests = 0
 248   failed = []
 249   disabled = []
 250
 251   for _, desc in parse_dsc.GenerateProjects(tree):
 252     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 253     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 254     valid_configs = set(configs) & set(desc_configs)
 255     for toolchain in sorted(valid_toolchains):
 256       for config in sorted(valid_configs):
 257         test_name = GetTestName(desc, toolchain, config)
 258         total_tests += 1
 259         if IsTestDisabled(desc, toolchain, config):
 260           disabled.append(test_name)
 261           continue
 262
 263         tests_run += 1
 264         success = RunTestWithGtestOutput(desc, toolchain, config,
 265                                          retry_on_failure_times)
 266         if not success:
 267           failed.append(test_name)
 268
 269   if failed:
 270     WriteBanner('FAILED TESTS')
 271     for test in failed:
 272       print '  %s failed.' % test
 273
 274   if disabled:
 275     WriteBanner('DISABLED TESTS')
 276     for test in disabled:
 277       print '  %s disabled.' % test
 278
 279   WriteHorizontalBar()
 280   print 'Tests run: %d/%d (%d disabled).' % (
 281       tests_run, total_tests, len(disabled))
 282   print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run)
 283
 284   success = len(failed) != 0
 285   return success
 286
 287
 288 def BuildAllTestsInTree(tree, toolchains, configs):
 289   for branch, desc in parse_dsc.GenerateProjects(tree):
 290     desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
 291     valid_toolchains = set(toolchains) & set(desc['TOOLS'])
 292     valid_configs = set(configs) & set(desc_configs)
 293     for toolchain in sorted(valid_toolchains):
 294       for config in sorted(valid_configs):
 295         name = '%s/%s' % (branch, desc['NAME'])
 296         build_projects.BuildProjectsBranch(pepperdir, name, deps=False,
 297                                            clean=False, config=config,
 298                                            args=['TOOLCHAIN=%s' % toolchain])
 299
 300
 301 def GetProjectTree(include):
 302   # Everything in src is a library, and cannot be run.
 303   exclude = {'DEST': 'src'}
 304   try:
 305     return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include,
 306                                      exclude=exclude)
 307   except parse_dsc.ValidationError as e:
 308     buildbot_common.ErrorExit(str(e))
 309
 310
 311 def main(args):
 312   parser = optparse.OptionParser()
 313   parser.add_option('-c', '--config',
 314       help='Choose configuration to run (Debug or Release).  Runs both '
 315            'by default', action='append')
 316   parser.add_option('-x', '--experimental',
 317       help='Run experimental projects', action='store_true')
 318   parser.add_option('-t', '--toolchain',
 319       help='Run using toolchain. Can be passed more than once.',
 320       action='append', default=[])
 321   parser.add_option('-d', '--dest',
 322       help='Select which destinations (project types) are valid.',
 323       action='append')
 324   parser.add_option('-b', '--build',
 325       help='Build each project before testing.', action='store_true')
 326   parser.add_option('--retry-times',
 327       help='Number of types to retry on failure (Default: %default)',
 328           type='int', default=1)
 329
 330   options, args = parser.parse_args(args[1:])
 331
 332   if not options.toolchain:
 333     options.toolchain = ['newlib', 'glibc', 'pnacl', 'host']
 334
 335   if 'host' in options.toolchain:
 336     options.toolchain.remove('host')
 337     options.toolchain.append(platform)
 338     print 'Adding platform: ' + platform
 339
 340   ValidateToolchains(options.toolchain)
 341
 342   include = {}
 343   if options.toolchain:
 344     include['TOOLS'] = options.toolchain
 345     print 'Filter by toolchain: ' + str(options.toolchain)
 346   if not options.experimental:
 347     include['EXPERIMENTAL'] = False
 348   if options.dest:
 349     include['DEST'] = options.dest
 350     print 'Filter by type: ' + str(options.dest)
 351   if args:
 352     include['NAME'] = args
 353     print 'Filter by name: ' + str(args)
 354   if not options.config:
 355     options.config = ALL_CONFIGS
 356
 357   project_tree = GetProjectTree(include)
 358   if options.build:
 359     BuildAllTestsInTree(project_tree, options.toolchain, options.config)
 360
 361   return RunAllTestsInTree(project_tree, options.toolchain, options.config,
 362                            options.retry_times)
 363
 364
 365 if __name__ == '__main__':
 366   script_name = os.path.basename(sys.argv[0])
 367   try:
 368     sys.exit(main(sys.argv))
 369   except parse_dsc.ValidationError as e:
 370     buildbot_common.ErrorExit('%s: %s' % (script_name, e))
 371   except KeyboardInterrupt:
 372     buildbot_common.ErrorExit('%s: interrupted' % script_name)