ppapi/native_client/tools/browser_tester/browser_tester.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import glob
   7 import optparse
   8 import os.path
   9 import socket
  10 import sys
  11 import thread
  12 import time
  13 import urllib
  14
  15 # Allow the import of third party modules
  16 script_dir = os.path.dirname(os.path.abspath(__file__))
  17 sys.path.insert(0, os.path.join(script_dir, '../../../../third_party/'))
  18 sys.path.insert(0, os.path.join(script_dir, '../../../../tools/valgrind/'))
  19 sys.path.insert(0, os.path.join(script_dir, '../../../../testing/'))
  20
  21 import browsertester.browserlauncher
  22 import browsertester.rpclistener
  23 import browsertester.server
  24
  25 import memcheck_analyze
  26
  27 import test_env
  28
  29 def BuildArgParser():
  30   usage = 'usage: %prog [options]'
  31   parser = optparse.OptionParser(usage)
  32
  33   parser.add_option('-p', '--port', dest='port', action='store', type='int',
  34                     default='0', help='The TCP port the server will bind to. '
  35                     'The default is to pick an unused port number.')
  36   parser.add_option('--browser_path', dest='browser_path', action='store',
  37                     type='string', default=None,
  38                     help='Use the browser located here.')
  39   parser.add_option('--map_file', dest='map_files', action='append',
  40                     type='string', nargs=2, default=[],
  41                     metavar='DEST SRC',
  42                     help='Add file SRC to be served from the HTTP server, '
  43                     'to be made visible under the path DEST.')
  44   parser.add_option('--serving_dir', dest='serving_dirs', action='append',
  45                     type='string', default=[],
  46                     metavar='DIRNAME',
  47                     help='Add directory DIRNAME to be served from the HTTP '
  48                     'server to be made visible under the root.')
  49   parser.add_option('--output_dir', dest='output_dir', action='store',
  50                     type='string', default=None,
  51                     metavar='DIRNAME',
  52                     help='Set directory DIRNAME to be the output directory '
  53                     'when POSTing data to the server. NOTE: if this flag is '
  54                     'not set, POSTs will fail.')
  55   parser.add_option('--test_arg', dest='test_args', action='append',
  56                     type='string', nargs=2, default=[],
  57                     metavar='KEY VALUE',
  58                     help='Parameterize the test with a key/value pair.')
  59   parser.add_option('--redirect_url', dest='map_redirects', action='append',
  60                     type='string', nargs=2, default=[],
  61                     metavar='DEST SRC',
  62                     help='Add a redirect to the HTTP server, '
  63                     'requests for SRC will result in a redirect (302) to DEST.')
  64   parser.add_option('-f', '--file', dest='files', action='append',
  65                     type='string', default=[],
  66                     metavar='FILENAME',
  67                     help='Add a file to serve from the HTTP server, to be '
  68                     'made visible in the root directory.  '
  69                     '"--file path/to/foo.html" is equivalent to '
  70                     '"--map_file foo.html path/to/foo.html"')
  71   parser.add_option('--mime_type', dest='mime_types', action='append',
  72                     type='string', nargs=2, default=[], metavar='DEST SRC',
  73                     help='Map file extension SRC to MIME type DEST when '
  74                     'serving it from the HTTP server.')
  75   parser.add_option('-u', '--url', dest='url', action='store',
  76                     type='string', default=None,
  77                     help='The webpage to load.')
  78   parser.add_option('--ppapi_plugin', dest='ppapi_plugin', action='store',
  79                     type='string', default=None,
  80                     help='Use the browser plugin located here.')
  81   parser.add_option('--ppapi_plugin_mimetype', dest='ppapi_plugin_mimetype',
  82                     action='store', type='string', default='application/x-nacl',
  83                     help='Associate this mimetype with the browser plugin. '
  84                     'Unused if --ppapi_plugin is not specified.')
  85   parser.add_option('--sel_ldr', dest='sel_ldr', action='store',
  86                     type='string', default=None,
  87                     help='Use the sel_ldr located here.')
  88   parser.add_option('--sel_ldr_bootstrap', dest='sel_ldr_bootstrap',
  89                     action='store', type='string', default=None,
  90                     help='Use the bootstrap loader located here.')
  91   parser.add_option('--irt_library', dest='irt_library', action='store',
  92                     type='string', default=None,
  93                     help='Use the integrated runtime (IRT) library '
  94                     'located here.')
  95   parser.add_option('--interactive', dest='interactive', action='store_true',
  96                     default=False, help='Do not quit after testing is done. '
  97                     'Handy for iterative development.  Disables timeout.')
  98   parser.add_option('--debug', dest='debug', action='store_true', default=False,
  99                     help='Request debugging output from browser.')
 100   parser.add_option('--timeout', dest='timeout', action='store', type='float',
 101                     default=5.0,
 102                     help='The maximum amount of time to wait, in seconds, for '
 103                     'the browser to make a request. The timer resets with each '
 104                     'request.')
 105   parser.add_option('--hard_timeout', dest='hard_timeout', action='store',
 106                     type='float', default=None,
 107                     help='The maximum amount of time to wait, in seconds, for '
 108                     'the entire test.  This will kill runaway tests. ')
 109   parser.add_option('--allow_404', dest='allow_404', action='store_true',
 110                     default=False,
 111                     help='Allow 404s to occur without failing the test.')
 112   parser.add_option('-b', '--bandwidth', dest='bandwidth', action='store',
 113                     type='float', default='0.0',
 114                     help='The amount of bandwidth (megabits / second) to '
 115                     'simulate between the client and the server. This used for '
 116                     'replies with file payloads. All other responses are '
 117                     'assumed to be short. Bandwidth values <= 0.0 are assumed '
 118                     'to mean infinite bandwidth.')
 119   parser.add_option('--extension', dest='browser_extensions', action='append',
 120                     type='string', default=[],
 121                     help='Load the browser extensions located at the list of '
 122                     'paths. Note: this currently only works with the Chrome '
 123                     'browser.')
 124   parser.add_option('--tool', dest='tool', action='store',
 125                     type='string', default=None,
 126                     help='Run tests under a tool.')
 127   parser.add_option('--browser_flag', dest='browser_flags', action='append',
 128                     type='string', default=[],
 129                     help='Additional flags for the chrome command.')
 130   parser.add_option('--enable_ppapi_dev', dest='enable_ppapi_dev',
 131                     action='store', type='int', default=1,
 132                     help='Enable/disable PPAPI Dev interfaces while testing.')
 133   parser.add_option('--nacl_exe_stdin', dest='nacl_exe_stdin',
 134                     type='string', default=None,
 135                     help='Redirect standard input of NaCl executable.')
 136   parser.add_option('--nacl_exe_stdout', dest='nacl_exe_stdout',
 137                     type='string', default=None,
 138                     help='Redirect standard output of NaCl executable.')
 139   parser.add_option('--nacl_exe_stderr', dest='nacl_exe_stderr',
 140                     type='string', default=None,
 141                     help='Redirect standard error of NaCl executable.')
 142   parser.add_option('--expect_browser_process_crash',
 143                     dest='expect_browser_process_crash',
 144                     action='store_true',
 145                     help='Do not signal a failure if the browser process '
 146                     'crashes')
 147   parser.add_option('--enable_crash_reporter', dest='enable_crash_reporter',
 148                     action='store_true', default=False,
 149                     help='Force crash reporting on.')
 150   parser.add_option('--enable_sockets', dest='enable_sockets',
 151                     action='store_true', default=False,
 152                     help='Pass --allow-nacl-socket-api=<host> to Chrome, where '
 153                     '<host> is the name of the browser tester\'s web server.')
 154
 155   return parser
 156
 157
 158 def ProcessToolLogs(options, logs_dir):
 159   if options.tool == 'memcheck':
 160     analyzer = memcheck_analyze.MemcheckAnalyzer('', use_gdb=True)
 161     logs_wildcard = 'xml.*'
 162   files = glob.glob(os.path.join(logs_dir, logs_wildcard))
 163   retcode = analyzer.Report(files, options.url)
 164   return retcode
 165
 166
 167 # An exception that indicates possible flake.
 168 class RetryTest(Exception):
 169   pass
 170
 171
 172 def DumpNetLog(netlog):
 173   sys.stdout.write('\n')
 174   if not os.path.isfile(netlog):
 175     sys.stdout.write('Cannot find netlog, did Chrome actually launch?\n')
 176   else:
 177     sys.stdout.write('Netlog exists (%d bytes).\n' % os.path.getsize(netlog))
 178     sys.stdout.write('Dumping it to stdout.\n\n\n')
 179     sys.stdout.write(open(netlog).read())
 180     sys.stdout.write('\n\n\n')
 181
 182
 183 # Try to discover the real IP address of this machine.  If we can't figure it
 184 # out, fall back to localhost.
 185 # A windows bug makes using the loopback interface flaky in rare cases.
 186 # http://code.google.com/p/chromium/issues/detail?id=114369
 187 def GetHostName():
 188   host = 'localhost'
 189   try:
 190     host = socket.gethostbyname(socket.gethostname())
 191   except Exception:
 192     pass
 193   if host == '0.0.0.0':
 194     host = 'localhost'
 195   return host
 196
 197
 198 def RunTestsOnce(url, options):
 199   # Set the default here so we're assured hard_timeout will be defined.
 200   # Tests, such as run_inbrowser_trusted_crash_in_startup_test, may not use the
 201   # RunFromCommand line entry point - and otherwise get stuck in an infinite
 202   # loop when something goes wrong and the hard timeout is not set.
 203   # http://code.google.com/p/chromium/issues/detail?id=105406
 204   if options.hard_timeout is None:
 205     options.hard_timeout = options.timeout * 4
 206
 207   options.files.append(os.path.join(script_dir, 'browserdata', 'nacltest.js'))
 208
 209   # Setup the environment with the setuid sandbox path.
 210   os.environ.update(test_env.get_sandbox_env(os.environ))
 211
 212   # Create server
 213   host = GetHostName()
 214   try:
 215     server = browsertester.server.Create(host, options.port)
 216   except Exception:
 217     sys.stdout.write('Could not bind %r, falling back to localhost.\n' % host)
 218     server = browsertester.server.Create('localhost', options.port)
 219
 220   # If port 0 has been requested, an arbitrary port will be bound so we need to
 221   # query it.  Older version of Python do not set server_address correctly when
 222   # The requested port is 0 so we need to break encapsulation and query the
 223   # socket directly.
 224   host, port = server.socket.getsockname()
 225
 226   file_mapping = dict(options.map_files)
 227   for filename in options.files:
 228     file_mapping[os.path.basename(filename)] = filename
 229   for server_path, real_path in file_mapping.iteritems():
 230     if not os.path.exists(real_path):
 231       raise AssertionError('\'%s\' does not exist.' % real_path)
 232   mime_types = {}
 233   for ext, mime_type in options.mime_types:
 234     mime_types['.' + ext] = mime_type
 235
 236   def ShutdownCallback():
 237     server.TestingEnded()
 238     close_browser = options.tool is not None and not options.interactive
 239     return close_browser
 240
 241   listener = browsertester.rpclistener.RPCListener(ShutdownCallback)
 242   server.Configure(file_mapping,
 243                    dict(options.map_redirects),
 244                    mime_types,
 245                    options.allow_404,
 246                    options.bandwidth,
 247                    listener,
 248                    options.serving_dirs,
 249                    options.output_dir)
 250
 251   browser = browsertester.browserlauncher.ChromeLauncher(options)
 252
 253   full_url = 'http://%s:%d/%s' % (host, port, url)
 254   if len(options.test_args) > 0:
 255     full_url += '?' + urllib.urlencode(options.test_args)
 256   browser.Run(full_url, host, port)
 257   server.TestingBegun(0.125)
 258
 259   # In Python 2.5, server.handle_request may block indefinitely.  Serving pages
 260   # is done in its own thread so the main thread can time out as needed.
 261   def Serve():
 262     while server.test_in_progress or options.interactive:
 263       server.handle_request()
 264   thread.start_new_thread(Serve, ())
 265
 266   tool_failed = False
 267   time_started = time.time()
 268
 269   def HardTimeout(total_time):
 270     return total_time >= 0.0 and time.time() - time_started >= total_time
 271
 272   try:
 273     while server.test_in_progress or options.interactive:
 274       if not browser.IsRunning():
 275         if options.expect_browser_process_crash:
 276           break
 277         listener.ServerError('Browser process ended during test '
 278                              '(return code %r)' % browser.GetReturnCode())
 279         # If Chrome exits prematurely without making a single request to the
 280         # web server, this is probally a Chrome crash-on-launch bug not related
 281         # to the test at hand.  Retry, unless we're in interactive mode.  In
 282         # interactive mode the user may manually close the browser, so don't
 283         # retry (it would just be annoying.)
 284         if not server.received_request and not options.interactive:
 285           raise RetryTest('Chrome failed to launch.')
 286         else:
 287           break
 288       elif not options.interactive and server.TimedOut(options.timeout):
 289         js_time = server.TimeSinceJSHeartbeat()
 290         err = 'Did not hear from the test for %.1f seconds.' % options.timeout
 291         err += '\nHeard from Javascript %.1f seconds ago.' % js_time
 292         if js_time > 2.0:
 293           err += '\nThe renderer probably hung or crashed.'
 294         else:
 295           err += '\nThe test probably did not get a callback that it expected.'
 296         listener.ServerError(err)
 297         if not server.received_request:
 298           raise RetryTest('Chrome hung before running the test.')
 299         break
 300       elif not options.interactive and HardTimeout(options.hard_timeout):
 301         listener.ServerError('The test took over %.1f seconds.  This is '
 302                              'probably a runaway test.' % options.hard_timeout)
 303         break
 304       else:
 305         # If Python 2.5 support is dropped, stick server.handle_request() here.
 306         time.sleep(0.125)
 307
 308     if options.tool:
 309       sys.stdout.write('##################### Waiting for the tool to exit\n')
 310       browser.WaitForProcessDeath()
 311       sys.stdout.write('##################### Processing tool logs\n')
 312       tool_failed = ProcessToolLogs(options, browser.tool_log_dir)
 313
 314   finally:
 315     try:
 316       if listener.ever_failed and not options.interactive:
 317         if not server.received_request:
 318           sys.stdout.write('\nNo URLs were served by the test runner. It is '
 319                            'unlikely this test failure has anything to do with '
 320                            'this particular test.\n')
 321           DumpNetLog(browser.NetLogName())
 322     except Exception:
 323       listener.ever_failed = 1
 324     # Try to let the browser clean itself up normally before killing it.
 325     sys.stdout.write('##################### Terminating the browser\n')
 326     browser.WaitForProcessDeath()
 327     if browser.IsRunning():
 328       sys.stdout.write('##################### TERM failed, KILLING\n')
 329     # Always call Cleanup; it kills the process, but also removes the
 330     # user-data-dir.
 331     browser.Cleanup()
 332     # We avoid calling server.server_close() here because it causes
 333     # the HTTP server thread to exit uncleanly with an EBADF error,
 334     # which adds noise to the logs (though it does not cause the test
 335     # to fail).  server_close() does not attempt to tell the server
 336     # loop to shut down before closing the socket FD it is
 337     # select()ing.  Since we are about to exit, we don't really need
 338     # to close the socket FD.
 339
 340   if tool_failed:
 341     return 2
 342   elif listener.ever_failed:
 343     return 1
 344   else:
 345     return 0
 346
 347
 348 # This is an entrypoint for tests that treat the browser tester as a Python
 349 # library rather than an opaque script.
 350 # (e.g. run_inbrowser_trusted_crash_in_startup_test)
 351 def Run(url, options):
 352   result = 1
 353   attempt = 1
 354   while True:
 355     try:
 356       result = RunTestsOnce(url, options)
 357       if result:
 358         # Currently (2013/11/15) nacl_integration is fairly flaky and there is
 359         # not enough time to look into it.  Retry if the test fails for any
 360         # reason.  Note that in general this test runner tries to only retry
 361         # when a known flake is encountered.  (See the other raise
 362         # RetryTest(..)s in this file.)  This blanket retry means that those
 363         # other cases could be removed without changing the behavior of the test
 364         # runner, but it is hoped that this blanket retry will eventually be
 365         # unnecessary and subsequently removed.  The more precise retries have
 366         # been left in place to preserve the knowledge.
 367         raise RetryTest('HACK retrying failed test.')
 368       break
 369     except RetryTest:
 370       # Only retry once.
 371       if attempt < 2:
 372         sys.stdout.write('\n@@@STEP_WARNINGS@@@\n')
 373         sys.stdout.write('WARNING: suspected flake, retrying test!\n\n')
 374         attempt += 1
 375         continue
 376       else:
 377         sys.stdout.write('\nWARNING: failed too many times, not retrying.\n\n')
 378         result = 1
 379         break
 380   return result
 381
 382
 383 def RunFromCommandLine():
 384   parser = BuildArgParser()
 385   options, args = parser.parse_args()
 386
 387   if len(args) != 0:
 388     print args
 389     parser.error('Invalid arguments')
 390
 391   # Validate the URL
 392   url = options.url
 393   if url is None:
 394     parser.error('Must specify a URL')
 395
 396   return Run(url, options)
 397
 398
 399 if __name__ == '__main__':
 400   sys.exit(RunFromCommandLine())