build/android/pylib/instrumentation/instrumentation_test_instance.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import pickle
   8 import re
   9 import sys
  10
  11 from pylib import cmd_helper
  12 from pylib import constants
  13 from pylib import flag_changer
  14 from pylib.base import base_test_result
  15 from pylib.base import test_instance
  16 from pylib.instrumentation import test_result
  17 from pylib.instrumentation import instrumentation_parser
  18 from pylib.utils import apk_helper
  19 from pylib.utils import md5sum
  20 from pylib.utils import proguard
  21
  22 sys.path.append(
  23     os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', 'common'))
  24 import unittest_util
  25
  26 # Ref: http://developer.android.com/reference/android/app/Activity.html
  27 _ACTIVITY_RESULT_CANCELED = 0
  28 _ACTIVITY_RESULT_OK = -1
  29
  30 _DEFAULT_ANNOTATIONS = [
  31     'Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
  32     'EnormousTest', 'IntegrationTest']
  33 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
  34 _PICKLE_FORMAT_VERSION = 10
  35
  36
  37 # TODO(jbudorick): Make these private class methods of
  38 # InstrumentationTestInstance once the instrumentation test_runner is
  39 # deprecated.
  40 def ParseAmInstrumentRawOutput(raw_output):
  41   """Parses the output of an |am instrument -r| call.
  42
  43   Args:
  44     raw_output: the output of an |am instrument -r| call as a list of lines
  45   Returns:
  46     A 3-tuple containing:
  47       - the instrumentation code as an integer
  48       - the instrumentation result as a list of lines
  49       - the instrumentation statuses received as a list of 2-tuples
  50         containing:
  51         - the status code as an integer
  52         - the bundle dump as a dict mapping string keys to a list of
  53           strings, one for each line.
  54   """
  55   parser = instrumentation_parser.InstrumentationParser(raw_output)
  56   statuses = list(parser.IterStatus())
  57   code, bundle = parser.GetResult()
  58   return (code, bundle, statuses)
  59
  60
  61 def GenerateTestResults(
  62     result_code, result_bundle, statuses, start_ms, duration_ms):
  63   """Generate test results from |statuses|.
  64
  65   Args:
  66     result_code: The overall status code as an integer.
  67     result_bundle: The summary bundle dump as a dict.
  68     statuses: A list of 2-tuples containing:
  69       - the status code as an integer
  70       - the bundle dump as a dict mapping string keys to string values
  71       Note that this is the same as the third item in the 3-tuple returned by
  72       |_ParseAmInstrumentRawOutput|.
  73     start_ms: The start time of the test in milliseconds.
  74     duration_ms: The duration of the test in milliseconds.
  75
  76   Returns:
  77     A list containing an instance of InstrumentationTestResult for each test
  78     parsed.
  79   """
  80
  81   results = []
  82
  83   current_result = None
  84
  85   for status_code, bundle in statuses:
  86     test_class = bundle.get('class', '')
  87     test_method = bundle.get('test', '')
  88     if test_class and test_method:
  89       test_name = '%s#%s' % (test_class, test_method)
  90     else:
  91       continue
  92
  93     if status_code == instrumentation_parser.STATUS_CODE_START:
  94       if current_result:
  95         results.append(current_result)
  96       current_result = test_result.InstrumentationTestResult(
  97           test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
  98     else:
  99       if status_code == instrumentation_parser.STATUS_CODE_OK:
 100         if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'):
 101           current_result.SetType(base_test_result.ResultType.SKIP)
 102         elif current_result.GetType() == base_test_result.ResultType.UNKNOWN:
 103           current_result.SetType(base_test_result.ResultType.PASS)
 104       else:
 105         if status_code not in (instrumentation_parser.STATUS_CODE_ERROR,
 106                                instrumentation_parser.STATUS_CODE_FAILURE):
 107           logging.error('Unrecognized status code %d. Handling as an error.',
 108                         status_code)
 109         current_result.SetType(base_test_result.ResultType.FAIL)
 110         if 'stack' in bundle:
 111           current_result.SetLog(bundle['stack'])
 112
 113   if current_result:
 114     if current_result.GetType() == base_test_result.ResultType.UNKNOWN:
 115       crashed = (result_code == _ACTIVITY_RESULT_CANCELED
 116                  and any(_NATIVE_CRASH_RE.search(l)
 117                          for l in result_bundle.itervalues()))
 118       if crashed:
 119         current_result.SetType(base_test_result.ResultType.CRASH)
 120
 121     results.append(current_result)
 122
 123   return results
 124
 125
 126 class InstrumentationTestInstance(test_instance.TestInstance):
 127
 128   def __init__(self, args, isolate_delegate, error_func):
 129     super(InstrumentationTestInstance, self).__init__()
 130
 131     self._apk_under_test = None
 132     self._package_info = None
 133     self._test_apk = None
 134     self._test_jar = None
 135     self._test_package = None
 136     self._test_runner = None
 137     self._test_support_apk = None
 138     self.__initializeApkAttributes(args, error_func)
 139
 140     self._data_deps = None
 141     self._isolate_abs_path = None
 142     self._isolate_delegate = None
 143     self._isolated_abs_path = None
 144     self._test_data = None
 145     self.__initializeDataDependencyAttributes(args, isolate_delegate)
 146
 147     self._annotations = None
 148     self._excluded_annotations = None
 149     self._test_filter = None
 150     self.__initializeTestFilterAttributes(args)
 151
 152     self._flags = None
 153     self.__initializeFlagAttributes(args)
 154
 155   def __initializeApkAttributes(self, args, error_func):
 156     if args.apk_under_test.endswith('.apk'):
 157       self._apk_under_test = args.apk_under_test
 158     else:
 159       self._apk_under_test = os.path.join(
 160           constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
 161           '%s.apk' % args.apk_under_test)
 162
 163     if not os.path.exists(self._apk_under_test):
 164       error_func('Unable to find APK under test: %s' % self._apk_under_test)
 165
 166     if args.test_apk.endswith('.apk'):
 167       test_apk_root = os.path.splitext(os.path.basename(args.test_apk))[0]
 168       self._test_apk = args.test_apk
 169     else:
 170       test_apk_root = args.test_apk
 171       self._test_apk = os.path.join(
 172           constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
 173           '%s.apk' % args.test_apk)
 174
 175     self._test_jar = os.path.join(
 176         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
 177         '%s.jar' % test_apk_root)
 178     self._test_support_apk = os.path.join(
 179         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
 180         '%sSupport.apk' % test_apk_root)
 181
 182     if not os.path.exists(self._test_apk):
 183       error_func('Unable to find test APK: %s' % self._test_apk)
 184     if not os.path.exists(self._test_jar):
 185       error_func('Unable to find test JAR: %s' % self._test_jar)
 186
 187     self._test_package = apk_helper.GetPackageName(self.test_apk)
 188     self._test_runner = apk_helper.GetInstrumentationName(self.test_apk)
 189
 190     self._package_info = None
 191     for package_info in constants.PACKAGE_INFO.itervalues():
 192       if self._test_package == package_info.test_package:
 193         self._package_info = package_info
 194     if not self._package_info:
 195       logging.warning('Unable to find package info for %s', self._test_package)
 196
 197   def __initializeDataDependencyAttributes(self, args, isolate_delegate):
 198     self._data_deps = []
 199     if args.isolate_file_path:
 200       self._isolate_abs_path = os.path.abspath(args.isolate_file_path)
 201       self._isolate_delegate = isolate_delegate
 202       self._isolated_abs_path = os.path.join(
 203           constants.GetOutDirectory(), '%s.isolated' % self._test_package)
 204     else:
 205       self._isolate_delegate = None
 206
 207     # TODO(jbudorick): Deprecate and remove --test-data once data dependencies
 208     # are fully converted to isolate.
 209     if args.test_data:
 210       logging.info('Data dependencies specified via --test-data')
 211       self._test_data = args.test_data
 212     else:
 213       self._test_data = None
 214
 215     if not self._isolate_delegate and not self._test_data:
 216       logging.warning('No data dependencies will be pushed.')
 217
 218   def __initializeTestFilterAttributes(self, args):
 219     self._test_filter = args.test_filter
 220
 221     def annotation_dict_element(a):
 222       a = a.split('=')
 223       return (a[0], a[1] if len(a) == 2 else None)
 224
 225     if args.annotation_str:
 226       self._annotations = dict(
 227           annotation_dict_element(a)
 228           for a in args.annotation_str.split(','))
 229     elif not self._test_filter:
 230       self._annotations = dict(
 231           annotation_dict_element(a)
 232           for a in _DEFAULT_ANNOTATIONS)
 233     else:
 234       self._annotations = {}
 235
 236     if args.exclude_annotation_str:
 237       self._excluded_annotations = dict(
 238           annotation_dict_element(a)
 239           for a in args.exclude_annotation_str.split(','))
 240     else:
 241       self._excluded_annotations = {}
 242
 243   def __initializeFlagAttributes(self, args):
 244     self._flags = ['--disable-fre', '--enable-test-intents']
 245     # TODO(jbudorick): Transition "--device-flags" to "--device-flags-file"
 246     if hasattr(args, 'device_flags') and args.device_flags:
 247       with open(args.device_flags) as device_flags_file:
 248         stripped_lines = (l.strip() for l in device_flags_file)
 249         self._flags.extend([flag for flag in stripped_lines if flag])
 250     if hasattr(args, 'device_flags_file') and args.device_flags_file:
 251       with open(args.device_flags_file) as device_flags_file:
 252         stripped_lines = (l.strip() for l in device_flags_file)
 253         self._flags.extend([flag for flag in stripped_lines if flag])
 254
 255   @property
 256   def suite(self):
 257     return 'instrumentation'
 258
 259   @property
 260   def apk_under_test(self):
 261     return self._apk_under_test
 262
 263   @property
 264   def flags(self):
 265     return self._flags
 266
 267   @property
 268   def package_info(self):
 269     return self._package_info
 270
 271   @property
 272   def test_apk(self):
 273     return self._test_apk
 274
 275   @property
 276   def test_jar(self):
 277     return self._test_jar
 278
 279   @property
 280   def test_support_apk(self):
 281     return self._test_support_apk
 282
 283   @property
 284   def test_package(self):
 285     return self._test_package
 286
 287   @property
 288   def test_runner(self):
 289     return self._test_runner
 290
 291   #override
 292   def TestType(self):
 293     return 'instrumentation'
 294
 295   #override
 296   def SetUp(self):
 297     if self._isolate_delegate:
 298       self._isolate_delegate.Remap(
 299           self._isolate_abs_path, self._isolated_abs_path)
 300       self._isolate_delegate.MoveOutputDeps()
 301       self._data_deps.extend([(constants.ISOLATE_DEPS_DIR, None)])
 302
 303     # TODO(jbudorick): Convert existing tests that depend on the --test-data
 304     # mechanism to isolate, then remove this.
 305     if self._test_data:
 306       for t in self._test_data:
 307         device_rel_path, host_rel_path = t.split(':')
 308         host_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, host_rel_path)
 309         self._data_deps.extend(
 310             [(host_abs_path,
 311               [None, 'chrome', 'test', 'data', device_rel_path])])
 312
 313   def GetDataDependencies(self):
 314     return self._data_deps
 315
 316   def GetTests(self):
 317     pickle_path = '%s-proguard.pickle' % self.test_jar
 318     try:
 319       tests = self._GetTestsFromPickle(pickle_path, self.test_jar)
 320     except self.ProguardPickleException as e:
 321       logging.info('Getting tests from JAR via proguard. (%s)' % str(e))
 322       tests = self._GetTestsFromProguard(self.test_jar)
 323       self._SaveTestsToPickle(pickle_path, self.test_jar, tests)
 324     return self._InflateTests(self._FilterTests(tests))
 325
 326   class ProguardPickleException(Exception):
 327     pass
 328
 329   def _GetTestsFromPickle(self, pickle_path, jar_path):
 330     if not os.path.exists(pickle_path):
 331       raise self.ProguardPickleException('%s does not exist.' % pickle_path)
 332     if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
 333       raise self.ProguardPickleException(
 334           '%s newer than %s.' % (jar_path, pickle_path))
 335
 336     with open(pickle_path, 'r') as pickle_file:
 337       pickle_data = pickle.loads(pickle_file.read())
 338     jar_md5, _ = md5sum.CalculateHostMd5Sums(jar_path)[0]
 339
 340     try:
 341       if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION:
 342         raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.')
 343       if pickle_data['JAR_MD5SUM'] != jar_md5:
 344         raise self.ProguardPickleException('JAR file MD5 sum differs.')
 345       return pickle_data['TEST_METHODS']
 346     except TypeError as e:
 347       logging.error(pickle_data)
 348       raise self.ProguardPickleException(str(e))
 349
 350   def _GetTestsFromProguard(self, jar_path):
 351     p = proguard.Dump(jar_path)
 352
 353     def is_test_class(c):
 354       return c['class'].endswith('Test')
 355
 356     def is_test_method(m):
 357       return m['method'].startswith('test')
 358
 359     class_lookup = dict((c['class'], c) for c in p['classes'])
 360     def recursive_get_class_annotations(c):
 361       s = c['superclass']
 362       if s in class_lookup:
 363         a = recursive_get_class_annotations(class_lookup[s])
 364       else:
 365         a = {}
 366       a.update(c['annotations'])
 367       return a
 368
 369     def stripped_test_class(c):
 370       return {
 371         'class': c['class'],
 372         'annotations': recursive_get_class_annotations(c),
 373         'methods': [m for m in c['methods'] if is_test_method(m)],
 374       }
 375
 376     return [stripped_test_class(c) for c in p['classes']
 377             if is_test_class(c)]
 378
 379   def _SaveTestsToPickle(self, pickle_path, jar_path, tests):
 380     jar_md5, _ = md5sum.CalculateHostMd5Sums(jar_path)[0]
 381     pickle_data = {
 382       'VERSION': _PICKLE_FORMAT_VERSION,
 383       'JAR_MD5SUM': jar_md5,
 384       'TEST_METHODS': tests,
 385     }
 386     with open(pickle_path, 'w') as pickle_file:
 387       pickle.dump(pickle_data, pickle_file)
 388
 389   def _FilterTests(self, tests):
 390
 391     def gtest_filter(c, m):
 392       t = ['%s.%s' % (c['class'].split('.')[-1], m['method'])]
 393       return (not self._test_filter
 394               or unittest_util.FilterTestNames(t, self._test_filter))
 395
 396     def annotation_filter(all_annotations):
 397       if not self._annotations:
 398         return True
 399       return any_annotation_matches(self._annotations, all_annotations)
 400
 401     def excluded_annotation_filter(all_annotations):
 402       if not self._excluded_annotations:
 403         return True
 404       return not any_annotation_matches(self._excluded_annotations,
 405                                         all_annotations)
 406
 407     def any_annotation_matches(annotations, all_annotations):
 408       return any(
 409           ak in all_annotations and (av is None or av == all_annotations[ak])
 410           for ak, av in annotations.iteritems())
 411
 412     filtered_classes = []
 413     for c in tests:
 414       filtered_methods = []
 415       for m in c['methods']:
 416         # Gtest filtering
 417         if not gtest_filter(c, m):
 418           continue
 419
 420         all_annotations = dict(c['annotations'])
 421         all_annotations.update(m['annotations'])
 422         if (not annotation_filter(all_annotations)
 423             or not excluded_annotation_filter(all_annotations)):
 424           continue
 425
 426         filtered_methods.append(m)
 427
 428       if filtered_methods:
 429         filtered_class = dict(c)
 430         filtered_class['methods'] = filtered_methods
 431         filtered_classes.append(filtered_class)
 432
 433     return filtered_classes
 434
 435   def _InflateTests(self, tests):
 436     inflated_tests = []
 437     for c in tests:
 438       for m in c['methods']:
 439         a = dict(c['annotations'])
 440         a.update(m['annotations'])
 441         inflated_tests.append({
 442             'class': c['class'],
 443             'method': m['method'],
 444             'annotations': a,
 445         })
 446     return inflated_tests
 447
 448   @staticmethod
 449   def ParseAmInstrumentRawOutput(raw_output):
 450     return ParseAmInstrumentRawOutput(raw_output)
 451
 452   @staticmethod
 453   def GenerateTestResults(
 454       result_code, result_bundle, statuses, start_ms, duration_ms):
 455     return GenerateTestResults(result_code, result_bundle, statuses,
 456                                start_ms, duration_ms)
 457
 458   #override
 459   def TearDown(self):
 460     if self._isolate_delegate:
 461       self._isolate_delegate.Clear()
 462