mojo/tools/mopy/gtest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import Queue
   8 import re
   9 import subprocess
  10 import sys
  11 import threading
  12 import time
  13
  14 from mopy.config import Config
  15 from mopy.paths import Paths
  16
  17 sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)),
  18                              '..', '..', '..', 'testing'))
  19 import xvfb
  20
  21
  22 # The DISPLAY ID number used for xvfb, incremented with each use.
  23 XVFB_DISPLAY_ID = 9
  24
  25
  26 def set_color():
  27   '''Run gtests with color on TTY, unless its environment variable is set.'''
  28   if sys.stdout.isatty() and 'GTEST_COLOR' not in os.environ:
  29     logging.getLogger().debug('Setting GTEST_COLOR=yes')
  30     os.environ['GTEST_COLOR'] = 'yes'
  31
  32
  33 def run_apptest(config, shell, args, apptest, isolate):
  34   '''Run the apptest; optionally isolating fixtures across shell invocations.
  35
  36   Returns the list of test fixtures run and the list of failed test fixtures.
  37   TODO(msw): Also return the list of DISABLED test fixtures.
  38
  39   Args:
  40     config: The mopy.config.Config for the build.
  41     shell: The mopy.android.AndroidShell, if Android is the target platform.
  42     args: The arguments for the shell or apptest.
  43     apptest: The application test URL.
  44     isolate: True if the test fixtures should be run in isolation.
  45   '''
  46   if not isolate:
  47     return _run_apptest_with_retry(config, shell, args, apptest)
  48
  49   fixtures = _get_fixtures(config, shell, args, apptest)
  50   fixtures = [f for f in fixtures if not f.startswith('DISABLED_')]
  51   failed = []
  52   for fixture in fixtures:
  53     arguments = args + ['--gtest_filter=%s' % fixture]
  54     failures = _run_apptest_with_retry(config, shell, arguments, apptest)[1]
  55     failed.extend(failures if failures != [apptest] else [fixture])
  56     # Abort when 20 fixtures, or a tenth of the apptest fixtures, have failed.
  57     # base::TestLauncher does this for timeouts and unknown results.
  58     if len(failed) >= max(20, len(fixtures) / 10):
  59       print 'Too many failing fixtures (%d), exiting now.' % len(failed)
  60       return (fixtures, failed + [apptest + ' aborted for excessive failures.'])
  61   return (fixtures, failed)
  62
  63
  64 # TODO(msw): Determine proper test retry counts; allow configuration.
  65 def _run_apptest_with_retry(config, shell, args, apptest, retry_count=2):
  66   '''Runs an apptest, retrying on failure; returns the fixtures and failures.'''
  67   (tests, failed) = _run_apptest(config, shell, args, apptest)
  68   while failed and retry_count:
  69     print 'Retrying failed tests (%d attempts remaining)' % retry_count
  70     arguments = args
  71     # Retry only the failing fixtures if there is no existing filter specified.
  72     if failed != [apptest] and not [a for a in args if '--gtest_filter=' in a]:
  73       arguments += ['--gtest_filter=%s' % ':'.join(failed)]
  74     failed = _run_apptest(config, shell, arguments, apptest)[1]
  75     retry_count -= 1
  76   return (tests, failed)
  77
  78
  79 def _run_apptest(config, shell, args, apptest):
  80   '''Runs an apptest; returns the list of fixtures and the list of failures.'''
  81   command = _build_command_line(config, args, apptest)
  82   logging.getLogger().debug('Command: %s' % ' '.join(command))
  83   start_time = time.time()
  84
  85   try:
  86     output = _run_test_with_xvfb(config, shell, args, apptest)
  87   except Exception as e:
  88     _print_exception(command, e)
  89     return ([apptest], [apptest])
  90
  91   # Find all fixtures begun from gtest's '[ RUN      ] <Suite.Fixture>' output.
  92   tests = [x for x in output.split('\n') if x.find('[ RUN      ] ') != -1]
  93   tests = [x.strip(' \t\n\r')[x.find('[ RUN      ] ') + 13:] for x in tests]
  94
  95   # Fail on output with gtest's '[  FAILED  ]' or a lack of '[       OK ]'.
  96   # The latter check ensures failure on broken command lines, hung output, etc.
  97   # Check output instead of exit codes because mojo shell always exits with 0.
  98   failed = [x for x in tests if (re.search('\[  FAILED  \].*' + x, output) or
  99                                  not re.search('\[       OK \].*' + x, output))]
 100
 101   ms = int(round(1000 * (time.time() - start_time)))
 102   if failed:
 103     _print_exception(command, output, ms)
 104   else:
 105     logging.getLogger().debug('Passed in %d ms with output:\n%s' % (ms, output))
 106   return (tests, failed)
 107
 108
 109 def _get_fixtures(config, shell, args, apptest):
 110   '''Returns an apptest's 'Suite.Fixture' list via --gtest_list_tests output.'''
 111   arguments = args + ['--gtest_list_tests']
 112   command = _build_command_line(config, arguments, apptest)
 113   logging.getLogger().debug('Command: %s' % ' '.join(command))
 114   try:
 115     tests = _run_test_with_xvfb(config, shell, arguments, apptest)
 116     logging.getLogger().debug('Tests for %s:\n%s' % (apptest, tests))
 117     # Remove log lines from the output and ensure it matches known formatting.
 118     tests = re.sub('^(\[|WARNING: linker:).*\n', '', tests, flags=re.MULTILINE)
 119     if not re.match('^(\w*\.\r?\n(  \w*\r?\n)+)+', tests):
 120       raise Exception('Unrecognized --gtest_list_tests output:\n%s' % tests)
 121     test_list = []
 122     for line in tests.split('\n'):
 123       if not line:
 124         continue
 125       if line[0] != ' ':
 126         suite = line.strip()
 127         continue
 128       test_list.append(suite + line.strip())
 129     return test_list
 130   except Exception as e:
 131     _print_exception(command, e)
 132   return []
 133
 134
 135 def _print_exception(command_line, exception, milliseconds=None):
 136   '''Print a formatted exception raised from a failed command execution.'''
 137   details = (' (in %d ms)' % milliseconds) if milliseconds else ''
 138   if hasattr(exception, 'returncode'):
 139     details += ' (with exit code %d)' % exception.returncode
 140   print '\n[  FAILED  ] Command%s: %s' % (details, ' '.join(command_line))
 141   print 72 * '-'
 142   if hasattr(exception, 'output'):
 143     print exception.output
 144   print str(exception)
 145   print 72 * '-'
 146
 147
 148 def _build_command_line(config, args, apptest):
 149   '''Build the apptest command line. This value isn't executed on Android.'''
 150   not_list_tests = not '--gtest_list_tests' in args
 151   data_dir = ['--use-temporary-user-data-dir'] if not_list_tests else []
 152   return Paths(config).mojo_runner + data_dir + args + [apptest]
 153
 154
 155 def _run_test_with_xvfb(config, shell, args, apptest):
 156   '''Run the test with xvfb; return the output or raise an exception.'''
 157   env = os.environ.copy()
 158   if (config.target_os != Config.OS_LINUX or '--gtest_list_tests' in args
 159       or not xvfb.should_start_xvfb(env)):
 160     return _run_test_with_timeout(config, shell, args, apptest, env)
 161
 162   try:
 163     # Simply prepending xvfb.py to the command line precludes direct control of
 164     # test subprocesses, and prevents easily getting output when tests timeout.
 165     xvfb_proc = None
 166     openbox_proc = None
 167     global XVFB_DISPLAY_ID
 168     display_string = ':' + str(XVFB_DISPLAY_ID)
 169     (xvfb_proc, openbox_proc) = xvfb.start_xvfb(env, Paths(config).build_dir,
 170                                                 display=display_string)
 171     XVFB_DISPLAY_ID = (XVFB_DISPLAY_ID + 1) % 50000
 172     if not xvfb_proc or not xvfb_proc.pid:
 173       raise Exception('Xvfb failed to start; aborting test run.')
 174     if not openbox_proc or not openbox_proc.pid:
 175       raise Exception('Openbox failed to start; aborting test run.')
 176     logging.getLogger().debug('Running Xvfb %s (pid %d) and Openbox (pid %d).' %
 177                               (display_string, xvfb_proc.pid, openbox_proc.pid))
 178     return _run_test_with_timeout(config, shell, args, apptest, env)
 179   finally:
 180     xvfb.kill(xvfb_proc)
 181     xvfb.kill(openbox_proc)
 182
 183
 184 # TODO(msw): Determine proper test timeout durations (starting small).
 185 def _run_test_with_timeout(config, shell, args, apptest, env, seconds=10):
 186   '''Run the test with a timeout; return the output or raise an exception.'''
 187   result = Queue.Queue()
 188   thread = threading.Thread(target=_run_test,
 189                             args=(config, shell, args, apptest, env, result))
 190   thread.start()
 191   process_or_shell = result.get()
 192   thread.join(seconds)
 193   timeout_exception = ''
 194
 195   if thread.is_alive():
 196     timeout_exception = '\nError: Test timeout after %s seconds' % seconds
 197     logging.getLogger().debug('Killing the runner or shell for timeout.')
 198     try:
 199       process_or_shell.kill()
 200     except OSError:
 201       pass  # The process may have ended after checking |is_alive|.
 202
 203   thread.join(seconds)
 204   if thread.is_alive():
 205     raise Exception('Error: Test hung and could not be killed!')
 206   if result.empty():
 207     raise Exception('Error: Test exited with no output.')
 208   (output, exception) = result.get()
 209   exception += timeout_exception
 210   if exception:
 211     raise Exception('%s%s%s' % (output, '\n' if output else '', exception))
 212   return output
 213
 214
 215 def _run_test(config, shell, args, apptest, env, result):
 216   '''Run the test; put the shell/proc, output and any exception in |result|.'''
 217   output = ''
 218   exception = ''
 219   try:
 220     if config.target_os != Config.OS_ANDROID:
 221       command = _build_command_line(config, args, apptest)
 222       process = subprocess.Popen(command, stdout=subprocess.PIPE,
 223                                  stderr=subprocess.PIPE, env=env)
 224       result.put(process)
 225       (output, stderr_output) = process.communicate()
 226       if process.returncode:
 227         exception = 'Error: Test exited with code: %d\n%s' % (
 228             process.returncode, stderr_output)
 229       elif config.is_verbose:
 230         output += '\n' + stderr_output
 231     else:
 232       assert shell
 233       result.put(shell)
 234       (r, w) = os.pipe()
 235       with os.fdopen(r, 'r') as rf:
 236         with os.fdopen(w, 'w') as wf:
 237           arguments = args + [apptest]
 238           shell.StartActivity('MojoShellActivity', arguments, wf, wf.close)
 239           output = rf.read()
 240   except Exception as e:
 241     output += (e.output + '\n') if hasattr(e, 'output') else ''
 242     exception += str(e)
 243   result.put((output, exception))