build/android/pylib/gtest/gtest_test_instance.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import re
   8 import sys
   9 import tempfile
  10
  11 from devil.android import apk_helper
  12 from pylib import constants
  13 from pylib.base import base_test_result
  14 from pylib.base import test_instance
  15
  16 sys.path.append(os.path.join(
  17     constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', 'common'))
  18 import unittest_util # pylint: disable=import-error
  19
  20
  21 BROWSER_TEST_SUITES = [
  22   'components_browsertests',
  23   'content_browsertests',
  24 ]
  25
  26
  27 _DEFAULT_ISOLATE_FILE_PATHS = {
  28     'base_unittests': 'base/base_unittests.isolate',
  29     'blink_heap_unittests':
  30       'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate',
  31     'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
  32     'cc_perftests': 'cc/cc_perftests.isolate',
  33     'components_browsertests': 'components/components_browsertests.isolate',
  34     'components_unittests': 'components/components_unittests.isolate',
  35     'content_browsertests': 'content/content_browsertests.isolate',
  36     'content_unittests': 'content/content_unittests.isolate',
  37     'media_perftests': 'media/media_perftests.isolate',
  38     'media_unittests': 'media/media_unittests.isolate',
  39     'midi_unittests': 'media/midi/midi_unittests.isolate',
  40     'net_unittests': 'net/net_unittests.isolate',
  41     'sql_unittests': 'sql/sql_unittests.isolate',
  42     'sync_unit_tests': 'sync/sync_unit_tests.isolate',
  43     'ui_base_unittests': 'ui/base/ui_base_tests.isolate',
  44     'unit_tests': 'chrome/unit_tests.isolate',
  45     'webkit_unit_tests':
  46       'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
  47 }
  48
  49
  50 # Used for filtering large data deps at a finer grain than what's allowed in
  51 # isolate files since pushing deps to devices is expensive.
  52 # Wildcards are allowed.
  53 _DEPS_EXCLUSION_LIST = [
  54     'chrome/test/data/extensions/api_test',
  55     'chrome/test/data/extensions/secure_shell',
  56     'chrome/test/data/firefox*',
  57     'chrome/test/data/gpu',
  58     'chrome/test/data/image_decoding',
  59     'chrome/test/data/import',
  60     'chrome/test/data/page_cycler',
  61     'chrome/test/data/perf',
  62     'chrome/test/data/pyauto_private',
  63     'chrome/test/data/safari_import',
  64     'chrome/test/data/scroll',
  65     'chrome/test/data/third_party',
  66     'third_party/hunspell_dictionaries/*.dic',
  67     # crbug.com/258690
  68     'webkit/data/bmp_decoder',
  69     'webkit/data/ico_decoder',
  70 ]
  71
  72
  73 _EXTRA_NATIVE_TEST_ACTIVITY = (
  74     'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
  75         'NativeTestActivity')
  76 _EXTRA_SHARD_SIZE_LIMIT = (
  77     'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
  78         'ShardSizeLimit')
  79
  80 # TODO(jbudorick): Remove these once we're no longer parsing stdout to generate
  81 # results.
  82 _RE_TEST_STATUS = re.compile(
  83     r'\[ +((?:RUN)|(?:FAILED)|(?:OK)) +\] ?([^ ]+)(?: \((\d+) ms\))?$')
  84 _RE_TEST_RUN_STATUS = re.compile(
  85     r'\[ +(PASSED|RUNNER_FAILED|CRASHED) \] ?[^ ]+')
  86
  87
  88 # TODO(jbudorick): Make this a class method of GtestTestInstance once
  89 # test_package_apk and test_package_exe are gone.
  90 def ParseGTestListTests(raw_list):
  91   """Parses a raw test list as provided by --gtest_list_tests.
  92
  93   Args:
  94     raw_list: The raw test listing with the following format:
  95
  96     IPCChannelTest.
  97       SendMessageInChannelConnected
  98     IPCSyncChannelTest.
  99       Simple
 100       DISABLED_SendWithTimeoutMixedOKAndTimeout
 101
 102   Returns:
 103     A list of all tests. For the above raw listing:
 104
 105     [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple,
 106      IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout]
 107   """
 108   ret = []
 109   current = ''
 110   for test in raw_list:
 111     if not test:
 112       continue
 113     if test[0] != ' ':
 114       test_case = test.split()[0]
 115       if test_case.endswith('.'):
 116         current = test_case
 117     elif not 'YOU HAVE' in test:
 118       test_name = test.split()[0]
 119       ret += [current + test_name]
 120   return ret
 121
 122
 123 class GtestTestInstance(test_instance.TestInstance):
 124
 125   def __init__(self, args, isolate_delegate, error_func):
 126     super(GtestTestInstance, self).__init__()
 127     # TODO(jbudorick): Support multiple test suites.
 128     if len(args.suite_name) > 1:
 129       raise ValueError('Platform mode currently supports only 1 gtest suite')
 130     self._suite = args.suite_name[0]
 131
 132     self._apk_path = os.path.join(
 133         constants.GetOutDirectory(), '%s_apk' % self._suite,
 134         '%s-debug.apk' % self._suite)
 135     self._exe_path = os.path.join(constants.GetOutDirectory(),
 136                                   self._suite)
 137     if not os.path.exists(self._apk_path):
 138       self._apk_path = None
 139       self._activity = None
 140       self._package = None
 141       self._runner = None
 142     else:
 143       helper = apk_helper.ApkHelper(self._apk_path)
 144       self._activity = helper.GetActivityName()
 145       self._package = helper.GetPackageName()
 146       self._runner = helper.GetInstrumentationName()
 147       self._permissions = helper.GetPermissions()
 148       self._extras = {
 149         _EXTRA_NATIVE_TEST_ACTIVITY: self._activity,
 150       }
 151       if self._suite in BROWSER_TEST_SUITES:
 152         self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1
 153
 154     if not os.path.exists(self._exe_path):
 155       self._exe_path = None
 156     if not self._apk_path and not self._exe_path:
 157       error_func('Could not find apk or executable for %s' % self._suite)
 158
 159     self._data_deps = []
 160     if args.test_filter:
 161       self._gtest_filter = args.test_filter
 162     elif args.test_filter_file:
 163       with open(args.test_filter_file, 'r') as f:
 164         self._gtest_filter = ':'.join(l.strip() for l in f)
 165     else:
 166       self._gtest_filter = None
 167
 168     if not args.isolate_file_path:
 169       default_isolate_file_path = _DEFAULT_ISOLATE_FILE_PATHS.get(self._suite)
 170       if default_isolate_file_path:
 171         args.isolate_file_path = os.path.join(
 172             constants.DIR_SOURCE_ROOT, default_isolate_file_path)
 173
 174     if args.isolate_file_path:
 175       self._isolate_abs_path = os.path.abspath(args.isolate_file_path)
 176       self._isolate_delegate = isolate_delegate
 177       self._isolated_abs_path = os.path.join(
 178           constants.GetOutDirectory(), '%s.isolated' % self._suite)
 179     else:
 180       logging.warning('No isolate file provided. No data deps will be pushed.')
 181       self._isolate_delegate = None
 182
 183     if args.app_data_files:
 184       self._app_data_files = args.app_data_files
 185       if args.app_data_file_dir:
 186         self._app_data_file_dir = args.app_data_file_dir
 187       else:
 188         self._app_data_file_dir = tempfile.mkdtemp()
 189         logging.critical('Saving app files to %s', self._app_data_file_dir)
 190     else:
 191       self._app_data_files = None
 192       self._app_data_file_dir = None
 193
 194   #override
 195   def TestType(self):
 196     return 'gtest'
 197
 198   #override
 199   def SetUp(self):
 200     """Map data dependencies via isolate."""
 201     if self._isolate_delegate:
 202       self._isolate_delegate.Remap(
 203           self._isolate_abs_path, self._isolated_abs_path)
 204       self._isolate_delegate.PurgeExcluded(_DEPS_EXCLUSION_LIST)
 205       self._isolate_delegate.MoveOutputDeps()
 206       dest_dir = None
 207       if self._suite == 'breakpad_unittests':
 208         dest_dir = '/data/local/tmp/'
 209       self._data_deps.extend([(constants.ISOLATE_DEPS_DIR, dest_dir)])
 210
 211
 212   def GetDataDependencies(self):
 213     """Returns the test suite's data dependencies.
 214
 215     Returns:
 216       A list of (host_path, device_path) tuples to push. If device_path is
 217       None, the client is responsible for determining where to push the file.
 218     """
 219     return self._data_deps
 220
 221   def FilterTests(self, test_list, disabled_prefixes=None):
 222     """Filters |test_list| based on prefixes and, if present, a filter string.
 223
 224     Args:
 225       test_list: The list of tests to filter.
 226       disabled_prefixes: A list of test prefixes to filter. Defaults to
 227         DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_
 228     Returns:
 229       A filtered list of tests to run.
 230     """
 231     gtest_filter_strings = [
 232         self._GenerateDisabledFilterString(disabled_prefixes)]
 233     if self._gtest_filter:
 234       gtest_filter_strings.append(self._gtest_filter)
 235
 236     filtered_test_list = test_list
 237     for gtest_filter_string in gtest_filter_strings:
 238       logging.debug('Filtering tests using: %s', gtest_filter_string)
 239       filtered_test_list = unittest_util.FilterTestNames(
 240           filtered_test_list, gtest_filter_string)
 241     return filtered_test_list
 242
 243   def _GenerateDisabledFilterString(self, disabled_prefixes):
 244     disabled_filter_items = []
 245
 246     if disabled_prefixes is None:
 247       disabled_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_', 'PRE_', 'MANUAL_']
 248     disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes]
 249     disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes]
 250
 251     disabled_tests_file_path = os.path.join(
 252         constants.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest',
 253         'filter', '%s_disabled' % self._suite)
 254     if disabled_tests_file_path and os.path.exists(disabled_tests_file_path):
 255       with open(disabled_tests_file_path) as disabled_tests_file:
 256         disabled_filter_items += [
 257             '%s' % l for l in (line.strip() for line in disabled_tests_file)
 258             if l and not l.startswith('#')]
 259
 260     return '*-%s' % ':'.join(disabled_filter_items)
 261
 262   # pylint: disable=no-self-use
 263   def ParseGTestOutput(self, output):
 264     """Parses raw gtest output and returns a list of results.
 265
 266     Args:
 267       output: A list of output lines.
 268     Returns:
 269       A list of base_test_result.BaseTestResults.
 270     """
 271     results = []
 272     for l in output:
 273       matcher = _RE_TEST_STATUS.match(l)
 274       if matcher:
 275         result_type = None
 276         if matcher.group(1) == 'OK':
 277           result_type = base_test_result.ResultType.PASS
 278         elif matcher.group(1) == 'FAILED':
 279           result_type = base_test_result.ResultType.FAIL
 280
 281         if result_type:
 282           test_name = matcher.group(2)
 283           duration = matcher.group(3) if matcher.group(3) else 0
 284           results.append(base_test_result.BaseTestResult(
 285               test_name, result_type, duration))
 286       logging.info(l)
 287     return results
 288
 289   #override
 290   def TearDown(self):
 291     """Clear the mappings created by SetUp."""
 292     if self._isolate_delegate:
 293       self._isolate_delegate.Clear()
 294
 295   @property
 296   def activity(self):
 297     return self._activity
 298
 299   @property
 300   def apk(self):
 301     return self._apk_path
 302
 303   @property
 304   def app_file_dir(self):
 305     return self._app_data_file_dir
 306
 307   @property
 308   def app_files(self):
 309     return self._app_data_files
 310
 311   @property
 312   def exe(self):
 313     return self._exe_path
 314
 315   @property
 316   def extras(self):
 317     return self._extras
 318
 319   @property
 320   def package(self):
 321     return self._package
 322
 323   @property
 324   def permissions(self):
 325     return self._permissions
 326
 327   @property
 328   def runner(self):
 329     return self._runner
 330
 331   @property
 332   def suite(self):
 333     return self._suite
 334