mojo/tools/mopy/gtest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import Queue
   8 import re
   9 import subprocess
  10 import sys
  11 import threading
  12 import time
  13
  14 from mopy.config import Config
  15 from mopy.paths import Paths
  16
  17
  18 def set_color():
  19   """Run gtests with color on TTY, unless its environment variable is set."""
  20   if sys.stdout.isatty() and "GTEST_COLOR" not in os.environ:
  21     logging.getLogger().debug("Setting GTEST_COLOR=yes")
  22     os.environ["GTEST_COLOR"] = "yes"
  23
  24
  25 def run_apptest(config, shell, args, apptest, isolate):
  26   """Run the apptest; optionally isolating fixtures across shell invocations.
  27
  28   Returns the list of tests run and the list of failures.
  29
  30   Args:
  31     config: The mopy.config.Config for the build.
  32     shell: The mopy.android.AndroidShell, if Android is the target platform.
  33     args: The arguments for the shell or apptest.
  34     apptest: The application test URL.
  35     isolate: True if the test fixtures should be run in isolation.
  36   """
  37   tests = [apptest]
  38   failed = []
  39   if not isolate:
  40     # TODO(msw): Parse fixture-granular successes and failures in this case.
  41     # TODO(msw): Retry fixtures that failed, not the entire apptest suite.
  42     if not _run_apptest_with_retry(config, shell, args, apptest):
  43       failed.append(apptest)
  44   else:
  45     tests = _get_fixtures(config, shell, args, apptest)
  46     for fixture in tests:
  47       arguments = args + ["--gtest_filter=%s" % fixture]
  48       if not _run_apptest_with_retry(config, shell, arguments, apptest):
  49         failed.append(fixture)
  50       # Abort when 20 fixtures, or a tenth of the apptest fixtures, have failed.
  51       # base::TestLauncher does this for timeouts and unknown results.
  52       if len(failed) >= max(20, len(tests) / 10):
  53         print "Too many failing fixtures (%d), exiting now." % len(failed)
  54         return (tests, failed + [apptest + " aborted for excessive failures."])
  55   return (tests, failed)
  56
  57
  58 # TODO(msw): Determine proper test retry counts; allow configuration.
  59 def _run_apptest_with_retry(config, shell, args, apptest, try_count=3):
  60   """Runs an apptest, retrying on failure; returns True if any run passed."""
  61   for try_number in range(try_count):
  62     if _run_apptest(config, shell, args, apptest):
  63       return True
  64     print "Failed %s/%s test run attempts." % (try_number + 1, try_count)
  65   return False
  66
  67
  68 def _run_apptest(config, shell, args, apptest):
  69   """Runs an apptest and checks the output for signs of gtest failure."""
  70   command = _build_command_line(config, args, apptest)
  71   logging.getLogger().debug("Command: %s" % " ".join(command))
  72   start_time = time.time()
  73
  74   try:
  75     output = _run_test_with_timeout(config, shell, args, apptest)
  76   except Exception as e:
  77     _print_exception(command, e)
  78     return False
  79
  80   # Fail on output with gtest's "[  FAILED  ]" or a lack of "[  PASSED  ]".
  81   # The latter condition ensures failure on broken command lines or output.
  82   # Check output instead of exit codes because mojo shell always exits with 0.
  83   if output.find("[  FAILED  ]") != -1 or output.find("[  PASSED  ]") == -1:
  84     _print_exception(command, output)
  85     return False
  86
  87   ms = int(round(1000 * (time.time() - start_time)))
  88   logging.getLogger().debug("Passed with output (%d ms):\n%s" % (ms, output))
  89   return True
  90
  91
  92 def _get_fixtures(config, shell, args, apptest):
  93   """Returns an apptest's "Suite.Fixture" list via --gtest_list_tests output."""
  94   arguments = args + ["--gtest_list_tests"]
  95   command = _build_command_line(config, arguments, apptest)
  96   logging.getLogger().debug("Command: %s" % " ".join(command))
  97   try:
  98     tests = _run_test_with_timeout(config, shell, arguments, apptest)
  99     logging.getLogger().debug("Tests for %s:\n%s" % (apptest, tests))
 100     # Remove log lines from the output and ensure it matches known formatting.
 101     tests = re.sub("^(\[|WARNING: linker:).*\n", "", tests, flags=re.MULTILINE)
 102     if not re.match("^(\w*\.\r?\n(  \w*\r?\n)+)+", tests):
 103       raise Exception("Unrecognized --gtest_list_tests output:\n%s" % tests)
 104     tests = tests.split("\n")
 105     test_list = []
 106     for line in tests:
 107       if not line:
 108         continue
 109       if line[0] != " ":
 110         suite = line.strip()
 111         continue
 112       test_list.append(suite + line.strip())
 113     return test_list
 114   except Exception as e:
 115     _print_exception(command, e)
 116   return []
 117
 118
 119 def _print_exception(command_line, exception):
 120   """Print a formatted exception raised from a failed command execution."""
 121   exit_code = ""
 122   if hasattr(exception, 'returncode'):
 123     exit_code = " (exit code %d)" % exception.returncode
 124   print "\n[  FAILED  ] Command%s: %s" % (exit_code, " ".join(command_line))
 125   print 72 * "-"
 126   if hasattr(exception, 'output'):
 127     print exception.output
 128   print str(exception)
 129   print 72 * "-"
 130
 131
 132 def _build_command_line(config, args, apptest):
 133   """Build the apptest command line. This value isn't executed on Android."""
 134   paths = Paths(config)
 135   # On Linux, always run tests with xvfb, but not for --gtest_list_tests.
 136   use_xvfb = (config.target_os == Config.OS_LINUX and
 137               not "--gtest_list_tests" in args)
 138   prefix = [paths.xvfb, paths.build_dir] if use_xvfb else []
 139   return prefix + [paths.mojo_runner] + args + [apptest]
 140
 141
 142 # TODO(msw): Determine proper test timeout durations (starting small).
 143 def _run_test_with_timeout(config, shell, args, apptest, timeout_in_seconds=10):
 144   """Run the test with a timeout; return the output or raise an exception."""
 145   result = Queue.Queue()
 146   thread = threading.Thread(target=_run_test,
 147                             args=(config, shell, args, apptest, result))
 148   thread.start()
 149   process_or_shell = result.get()
 150   thread.join(timeout_in_seconds)
 151
 152   if thread.is_alive():
 153     try:
 154       process_or_shell.kill()
 155     except OSError:
 156       pass  # The process may have ended after checking |is_alive|.
 157     return "Error: Test timeout after %s seconds" % timeout_in_seconds
 158
 159   if not result.empty():
 160     (output, exception) = result.get()
 161     if exception:
 162       raise Exception("%s%s%s" % (output, "\n" if output else "", exception))
 163     return output
 164
 165   return "Error: Test exited with no output."
 166
 167
 168 def _run_test(config, shell, args, apptest, result):
 169   """Run the test and put the output and any exception in |result|."""
 170   output = ""
 171   exception = ""
 172   try:
 173     if (config.target_os != Config.OS_ANDROID):
 174       command = _build_command_line(config, args, apptest)
 175       process = subprocess.Popen(command, stdout=subprocess.PIPE,
 176                                  stderr=subprocess.PIPE)
 177       result.put(process)
 178       process.wait()
 179       if not process.poll():
 180         output = str(process.stdout.read())
 181       else:
 182         exception = "Error: Test exited with code: %d" % process.returncode
 183     else:
 184       assert shell
 185       result.put(shell)
 186       (r, w) = os.pipe()
 187       with os.fdopen(r, "r") as rf:
 188         with os.fdopen(w, "w") as wf:
 189           arguments = args + [apptest]
 190           shell.StartActivity('MojoShellActivity', arguments, wf, wf.close)
 191           output = rf.read()
 192   except Exception as e:
 193     output = e.output if hasattr(e, 'output') else ""
 194     exception = str(e)
 195   result.put((output, exception))