mojo/tools/mopy/gtest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import Queue
   8 import re
   9 import subprocess
  10 import sys
  11 import threading
  12 import time
  13
  14 from mopy.config import Config
  15 from mopy.paths import Paths
  16
  17 sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),
  18                              '..', '..', '..', 'testing'))
  19 import xvfb
  20
  21
  22 # The DISPLAY ID number used for xvfb, incremented with each use.
  23 XVFB_DISPLAY_ID = 9
  24
  25
  26 def set_color():
  27   '''Run gtests with color on TTY, unless its environment variable is set.'''
  28   if sys.stdout.isatty() and 'GTEST_COLOR' not in os.environ:
  29     logging.getLogger().debug('Setting GTEST_COLOR=yes')
  30     os.environ['GTEST_COLOR'] = 'yes'
  31
  32
  33 def run_apptest(config, shell, args, apptest, isolate):
  34   '''Run the apptest; optionally isolating fixtures across shell invocations.
  35
  36   Returns the list of tests run and the list of failures.
  37
  38   Args:
  39     config: The mopy.config.Config for the build.
  40     shell: The mopy.android.AndroidShell, if Android is the target platform.
  41     args: The arguments for the shell or apptest.
  42     apptest: The application test URL.
  43     isolate: True if the test fixtures should be run in isolation.
  44   '''
  45   tests = [apptest]
  46   failed = []
  47   if not isolate:
  48     # TODO(msw): Parse fixture-granular successes and failures in this case.
  49     # TODO(msw): Retry fixtures that failed, not the entire apptest suite.
  50     if not _run_apptest_with_retry(config, shell, args, apptest):
  51       failed.append(apptest)
  52   else:
  53     tests = _get_fixtures(config, shell, args, apptest)
  54     for fixture in tests:
  55       arguments = args + ['--gtest_filter=%s' % fixture]
  56       if not _run_apptest_with_retry(config, shell, arguments, apptest):
  57         failed.append(fixture)
  58       # Abort when 20 fixtures, or a tenth of the apptest fixtures, have failed.
  59       # base::TestLauncher does this for timeouts and unknown results.
  60       if len(failed) >= max(20, len(tests) / 10):
  61         print 'Too many failing fixtures (%d), exiting now.' % len(failed)
  62         return (tests, failed + [apptest + ' aborted for excessive failures.'])
  63   return (tests, failed)
  64
  65
  66 # TODO(msw): Determine proper test retry counts; allow configuration.
  67 def _run_apptest_with_retry(config, shell, args, apptest, try_count=3):
  68   '''Runs an apptest, retrying on failure; returns True if any run passed.'''
  69   for try_number in range(try_count):
  70     if _run_apptest(config, shell, args, apptest):
  71       return True
  72     print 'Failed %s/%s test run attempts.' % (try_number + 1, try_count)
  73   return False
  74
  75
  76 def _run_apptest(config, shell, args, apptest):
  77   '''Runs an apptest and checks the output for signs of gtest failure.'''
  78   command = _build_command_line(config, args, apptest)
  79   logging.getLogger().debug('Command: %s' % ' '.join(command))
  80   start_time = time.time()
  81
  82   try:
  83     output = _run_test_with_xvfb(config, shell, args, apptest)
  84   except Exception as e:
  85     _print_exception(command, e)
  86     return False
  87
  88   # Fail on output with gtest's '[  FAILED  ]' or a lack of '[  PASSED  ]'.
  89   # The latter condition ensures failure on broken command lines or output.
  90   # Check output instead of exit codes because mojo shell always exits with 0.
  91   if output.find('[  FAILED  ]') != -1 or output.find('[  PASSED  ]') == -1:
  92     _print_exception(command, output)
  93     return False
  94
  95   ms = int(round(1000 * (time.time() - start_time)))
  96   logging.getLogger().debug('Passed with output (%d ms):\n%s' % (ms, output))
  97   return True
  98
  99
 100 def _get_fixtures(config, shell, args, apptest):
 101   '''Returns an apptest's 'Suite.Fixture' list via --gtest_list_tests output.'''
 102   arguments = args + ['--gtest_list_tests']
 103   command = _build_command_line(config, arguments, apptest)
 104   logging.getLogger().debug('Command: %s' % ' '.join(command))
 105   try:
 106     tests = _run_test_with_xvfb(config, shell, arguments, apptest)
 107     logging.getLogger().debug('Tests for %s:\n%s' % (apptest, tests))
 108     # Remove log lines from the output and ensure it matches known formatting.
 109     tests = re.sub('^(\[|WARNING: linker:).*\n', '', tests, flags=re.MULTILINE)
 110     if not re.match('^(\w*\.\r?\n(  \w*\r?\n)+)+', tests):
 111       raise Exception('Unrecognized --gtest_list_tests output:\n%s' % tests)
 112     tests = tests.split('\n')
 113     test_list = []
 114     for line in tests:
 115       if not line:
 116         continue
 117       if line[0] != ' ':
 118         suite = line.strip()
 119         continue
 120       test_list.append(suite + line.strip())
 121     return test_list
 122   except Exception as e:
 123     _print_exception(command, e)
 124   return []
 125
 126
 127 def _print_exception(command_line, exception):
 128   '''Print a formatted exception raised from a failed command execution.'''
 129   exit_code = ''
 130   if hasattr(exception, 'returncode'):
 131     exit_code = ' (exit code %d)' % exception.returncode
 132   print '\n[  FAILED  ] Command%s: %s' % (exit_code, ' '.join(command_line))
 133   print 72 * '-'
 134   if hasattr(exception, 'output'):
 135     print exception.output
 136   print str(exception)
 137   print 72 * '-'
 138
 139
 140 def _build_command_line(config, args, apptest):
 141   '''Build the apptest command line. This value isn't executed on Android.'''
 142   not_list_tests = not '--gtest_list_tests' in args
 143   data_dir = ['--use-temporary-user-data-dir'] if not_list_tests else []
 144   return [Paths(config).mojo_runner] + data_dir + args + [apptest]
 145
 146
 147 def _run_test_with_xvfb(config, shell, args, apptest):
 148   '''Run the test with xvfb; return the output or raise an exception.'''
 149   env = os.environ.copy()
 150   if (config.target_os != Config.OS_LINUX or '--gtest_list_tests' in args
 151       or not xvfb.should_start_xvfb(env)):
 152     return _run_test_with_timeout(config, shell, args, apptest, env)
 153
 154   try:
 155     # Simply prepending xvfb.py to the command line precludes direct control of
 156     # test subprocesses, and prevents easily getting output when tests timeout.
 157     xvfb_proc = None
 158     openbox_proc = None
 159     global XVFB_DISPLAY_ID
 160     display_string = ':' + str(XVFB_DISPLAY_ID)
 161     (xvfb_proc, openbox_proc) = xvfb.start_xvfb(env, Paths(config).build_dir,
 162                                                 display=display_string)
 163     XVFB_DISPLAY_ID = (XVFB_DISPLAY_ID + 1) % 50000
 164     if not xvfb_proc or not xvfb_proc.pid:
 165       raise Exception('Xvfb failed to start; aborting test run.')
 166     if not openbox_proc or not openbox_proc.pid:
 167       raise Exception('Openbox failed to start; aborting test run.')
 168     logging.getLogger().debug('Running Xvfb %s (pid %d) and Openbox (pid %d).' %
 169                               (display_string, xvfb_proc.pid, openbox_proc.pid))
 170     return _run_test_with_timeout(config, shell, args, apptest, env)
 171   finally:
 172     xvfb.kill(xvfb_proc)
 173     xvfb.kill(openbox_proc)
 174
 175
 176 # TODO(msw): Determine proper test timeout durations (starting small).
 177 def _run_test_with_timeout(config, shell, args, apptest, env, seconds=10):
 178   '''Run the test with a timeout; return the output or raise an exception.'''
 179   result = Queue.Queue()
 180   thread = threading.Thread(target=_run_test,
 181                             args=(config, shell, args, apptest, env, result))
 182   thread.start()
 183   process_or_shell = result.get()
 184   thread.join(seconds)
 185   timeout_exception = ''
 186
 187   if thread.is_alive():
 188     timeout_exception = '\nError: Test timeout after %s seconds' % seconds
 189     logging.getLogger().debug('Killing the runner or shell for timeout.')
 190     try:
 191       process_or_shell.kill()
 192     except OSError:
 193       pass  # The process may have ended after checking |is_alive|.
 194
 195   thread.join(seconds)
 196   if thread.is_alive():
 197     raise Exception('Error: Test hung and could not be killed!')
 198   if result.empty():
 199     raise Exception('Error: Test exited with no output.')
 200   (output, exception) = result.get()
 201   exception += timeout_exception
 202   if exception:
 203     raise Exception('%s%s%s' % (output, '\n' if output else '', exception))
 204   return output
 205
 206
 207 def _run_test(config, shell, args, apptest, env, result):
 208   '''Run the test; put the shell/proc, output and any exception in |result|.'''
 209   output = ''
 210   exception = ''
 211   try:
 212     if config.target_os != Config.OS_ANDROID:
 213       command = _build_command_line(config, args, apptest)
 214       process = subprocess.Popen(command, stdout=subprocess.PIPE,
 215                                  stderr=subprocess.PIPE, env=env)
 216       result.put(process)
 217       output = process.communicate()[0]
 218       if process.returncode:
 219         exception = 'Error: Test exited with code: %d' % process.returncode
 220     else:
 221       assert shell
 222       result.put(shell)
 223       (r, w) = os.pipe()
 224       with os.fdopen(r, 'r') as rf:
 225         with os.fdopen(w, 'w') as wf:
 226           arguments = args + [apptest]
 227           shell.StartActivity('MojoShellActivity', arguments, wf, wf.close)
 228           output = rf.read()
 229   except Exception as e:
 230     output += (e.output + '\n') if hasattr(e, 'output') else ''
 231     exception += str(e)
 232   result.put((output, exception))