build/android/pylib/instrumentation/instrumentation_test_instance.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import pickle
   8 import re
   9 import sys
  10
  11 from devil.android import apk_helper
  12 from devil.android import md5sum
  13 from pylib import constants
  14 from pylib.base import base_test_result
  15 from pylib.base import test_instance
  16 from pylib.instrumentation import test_result
  17 from pylib.instrumentation import instrumentation_parser
  18 from pylib.utils import proguard
  19
  20 sys.path.append(
  21     os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', 'common'))
  22 import unittest_util # pylint: disable=import-error
  23
  24 # Ref: http://developer.android.com/reference/android/app/Activity.html
  25 _ACTIVITY_RESULT_CANCELED = 0
  26 _ACTIVITY_RESULT_OK = -1
  27
  28 _DEFAULT_ANNOTATIONS = [
  29     'Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
  30     'EnormousTest', 'IntegrationTest']
  31 _EXTRA_ENABLE_HTTP_SERVER = (
  32     'org.chromium.chrome.test.ChromeInstrumentationTestRunner.'
  33         + 'EnableTestHttpServer')
  34 _EXTRA_DRIVER_TEST_LIST = (
  35     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
  36 _EXTRA_DRIVER_TEST_LIST_FILE = (
  37     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile')
  38 _EXTRA_DRIVER_TARGET_PACKAGE = (
  39     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage')
  40 _EXTRA_DRIVER_TARGET_CLASS = (
  41     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass')
  42 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
  43 _PICKLE_FORMAT_VERSION = 10
  44
  45
  46 # TODO(jbudorick): Make these private class methods of
  47 # InstrumentationTestInstance once the instrumentation test_runner is
  48 # deprecated.
  49 def ParseAmInstrumentRawOutput(raw_output):
  50   """Parses the output of an |am instrument -r| call.
  51
  52   Args:
  53     raw_output: the output of an |am instrument -r| call as a list of lines
  54   Returns:
  55     A 3-tuple containing:
  56       - the instrumentation code as an integer
  57       - the instrumentation result as a list of lines
  58       - the instrumentation statuses received as a list of 2-tuples
  59         containing:
  60         - the status code as an integer
  61         - the bundle dump as a dict mapping string keys to a list of
  62           strings, one for each line.
  63   """
  64   parser = instrumentation_parser.InstrumentationParser(raw_output)
  65   statuses = list(parser.IterStatus())
  66   code, bundle = parser.GetResult()
  67   return (code, bundle, statuses)
  68
  69
  70 def GenerateTestResults(
  71     result_code, result_bundle, statuses, start_ms, duration_ms):
  72   """Generate test results from |statuses|.
  73
  74   Args:
  75     result_code: The overall status code as an integer.
  76     result_bundle: The summary bundle dump as a dict.
  77     statuses: A list of 2-tuples containing:
  78       - the status code as an integer
  79       - the bundle dump as a dict mapping string keys to string values
  80       Note that this is the same as the third item in the 3-tuple returned by
  81       |_ParseAmInstrumentRawOutput|.
  82     start_ms: The start time of the test in milliseconds.
  83     duration_ms: The duration of the test in milliseconds.
  84
  85   Returns:
  86     A list containing an instance of InstrumentationTestResult for each test
  87     parsed.
  88   """
  89
  90   results = []
  91
  92   current_result = None
  93
  94   for status_code, bundle in statuses:
  95     test_class = bundle.get('class', '')
  96     test_method = bundle.get('test', '')
  97     if test_class and test_method:
  98       test_name = '%s#%s' % (test_class, test_method)
  99     else:
 100       continue
 101
 102     if status_code == instrumentation_parser.STATUS_CODE_START:
 103       if current_result:
 104         results.append(current_result)
 105       current_result = test_result.InstrumentationTestResult(
 106           test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
 107     else:
 108       if status_code == instrumentation_parser.STATUS_CODE_OK:
 109         if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'):
 110           current_result.SetType(base_test_result.ResultType.SKIP)
 111         elif current_result.GetType() == base_test_result.ResultType.UNKNOWN:
 112           current_result.SetType(base_test_result.ResultType.PASS)
 113       else:
 114         if status_code not in (instrumentation_parser.STATUS_CODE_ERROR,
 115                                instrumentation_parser.STATUS_CODE_FAILURE):
 116           logging.error('Unrecognized status code %d. Handling as an error.',
 117                         status_code)
 118         current_result.SetType(base_test_result.ResultType.FAIL)
 119         if 'stack' in bundle:
 120           current_result.SetLog(bundle['stack'])
 121
 122   if current_result:
 123     if current_result.GetType() == base_test_result.ResultType.UNKNOWN:
 124       crashed = (result_code == _ACTIVITY_RESULT_CANCELED
 125                  and any(_NATIVE_CRASH_RE.search(l)
 126                          for l in result_bundle.itervalues()))
 127       if crashed:
 128         current_result.SetType(base_test_result.ResultType.CRASH)
 129
 130     results.append(current_result)
 131
 132   return results
 133
 134
 135 class InstrumentationTestInstance(test_instance.TestInstance):
 136
 137   def __init__(self, args, isolate_delegate, error_func):
 138     super(InstrumentationTestInstance, self).__init__()
 139
 140     self._apk_under_test = None
 141     self._apk_under_test_permissions = None
 142     self._package_info = None
 143     self._suite = None
 144     self._test_apk = None
 145     self._test_jar = None
 146     self._test_package = None
 147     self._test_permissions = None
 148     self._test_runner = None
 149     self._test_support_apk = None
 150     self._initializeApkAttributes(args, error_func)
 151
 152     self._data_deps = None
 153     self._isolate_abs_path = None
 154     self._isolate_delegate = None
 155     self._isolated_abs_path = None
 156     self._test_data = None
 157     self._initializeDataDependencyAttributes(args, isolate_delegate)
 158
 159     self._annotations = None
 160     self._excluded_annotations = None
 161     self._test_filter = None
 162     self._initializeTestFilterAttributes(args)
 163
 164     self._flags = None
 165     self._initializeFlagAttributes(args)
 166
 167     self._driver_apk = None
 168     self._driver_package = None
 169     self._driver_name = None
 170     self._initializeDriverAttributes()
 171
 172   def _initializeApkAttributes(self, args, error_func):
 173     if args.apk_under_test.endswith('.apk'):
 174       self._apk_under_test = args.apk_under_test
 175     else:
 176       self._apk_under_test = os.path.join(
 177           constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
 178           '%s.apk' % args.apk_under_test)
 179
 180     if not os.path.exists(self._apk_under_test):
 181       error_func('Unable to find APK under test: %s' % self._apk_under_test)
 182
 183     apk = apk_helper.ApkHelper(self._apk_under_test)
 184     self._apk_under_test_permissions = apk.GetPermissions()
 185
 186     if args.test_apk.endswith('.apk'):
 187       self._suite = os.path.splitext(os.path.basename(args.test_apk))[0]
 188       self._test_apk = args.test_apk
 189     else:
 190       self._suite = args.test_apk
 191       self._test_apk = os.path.join(
 192           constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
 193           '%s.apk' % args.test_apk)
 194
 195     self._test_jar = os.path.join(
 196         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
 197         '%s.jar' % self._suite)
 198     self._test_support_apk = os.path.join(
 199         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
 200         '%sSupport.apk' % self._suite)
 201
 202     if not os.path.exists(self._test_apk):
 203       error_func('Unable to find test APK: %s' % self._test_apk)
 204     if not os.path.exists(self._test_jar):
 205       error_func('Unable to find test JAR: %s' % self._test_jar)
 206
 207     apk = apk_helper.ApkHelper(self.test_apk)
 208     self._test_package = apk.GetPackageName()
 209     self._test_permissions = apk.GetPermissions()
 210     self._test_runner = apk.GetInstrumentationName()
 211
 212     self._package_info = None
 213     for package_info in constants.PACKAGE_INFO.itervalues():
 214       if self._test_package == package_info.test_package:
 215         self._package_info = package_info
 216     if not self._package_info:
 217       logging.warning('Unable to find package info for %s', self._test_package)
 218
 219   def _initializeDataDependencyAttributes(self, args, isolate_delegate):
 220     self._data_deps = []
 221     if args.isolate_file_path:
 222       self._isolate_abs_path = os.path.abspath(args.isolate_file_path)
 223       self._isolate_delegate = isolate_delegate
 224       self._isolated_abs_path = os.path.join(
 225           constants.GetOutDirectory(), '%s.isolated' % self._test_package)
 226     else:
 227       self._isolate_delegate = None
 228
 229     # TODO(jbudorick): Deprecate and remove --test-data once data dependencies
 230     # are fully converted to isolate.
 231     if args.test_data:
 232       logging.info('Data dependencies specified via --test-data')
 233       self._test_data = args.test_data
 234     else:
 235       self._test_data = None
 236
 237     if not self._isolate_delegate and not self._test_data:
 238       logging.warning('No data dependencies will be pushed.')
 239
 240   def _initializeTestFilterAttributes(self, args):
 241     self._test_filter = args.test_filter
 242
 243     def annotation_dict_element(a):
 244       a = a.split('=')
 245       return (a[0], a[1] if len(a) == 2 else None)
 246
 247     if args.annotation_str:
 248       self._annotations = dict(
 249           annotation_dict_element(a)
 250           for a in args.annotation_str.split(','))
 251     elif not self._test_filter:
 252       self._annotations = dict(
 253           annotation_dict_element(a)
 254           for a in _DEFAULT_ANNOTATIONS)
 255     else:
 256       self._annotations = {}
 257
 258     if args.exclude_annotation_str:
 259       self._excluded_annotations = dict(
 260           annotation_dict_element(a)
 261           for a in args.exclude_annotation_str.split(','))
 262     else:
 263       self._excluded_annotations = {}
 264
 265   def _initializeFlagAttributes(self, args):
 266     self._flags = ['--disable-fre', '--enable-test-intents']
 267     # TODO(jbudorick): Transition "--device-flags" to "--device-flags-file"
 268     if hasattr(args, 'device_flags') and args.device_flags:
 269       with open(args.device_flags) as device_flags_file:
 270         stripped_lines = (l.strip() for l in device_flags_file)
 271         self._flags.extend([flag for flag in stripped_lines if flag])
 272     if hasattr(args, 'device_flags_file') and args.device_flags_file:
 273       with open(args.device_flags_file) as device_flags_file:
 274         stripped_lines = (l.strip() for l in device_flags_file)
 275         self._flags.extend([flag for flag in stripped_lines if flag])
 276
 277   def _initializeDriverAttributes(self):
 278     self._driver_apk = os.path.join(
 279         constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
 280         'OnDeviceInstrumentationDriver.apk')
 281     if os.path.exists(self._driver_apk):
 282       driver_apk = apk_helper.ApkHelper(self._driver_apk)
 283       self._driver_package = driver_apk.GetPackageName()
 284       self._driver_name = driver_apk.GetInstrumentationName()
 285     else:
 286       self._driver_apk = None
 287
 288   @property
 289   def apk_under_test(self):
 290     return self._apk_under_test
 291
 292   @property
 293   def apk_under_test_permissions(self):
 294    return self._apk_under_test_permissions
 295
 296   @property
 297   def flags(self):
 298     return self._flags
 299
 300   @property
 301   def driver_apk(self):
 302     return self._driver_apk
 303
 304   @property
 305   def driver_package(self):
 306     return self._driver_package
 307
 308   @property
 309   def driver_name(self):
 310     return self._driver_name
 311
 312   @property
 313   def package_info(self):
 314     return self._package_info
 315
 316   @property
 317   def suite(self):
 318     return self._suite
 319
 320   @property
 321   def test_apk(self):
 322     return self._test_apk
 323
 324   @property
 325   def test_jar(self):
 326     return self._test_jar
 327
 328   @property
 329   def test_support_apk(self):
 330     return self._test_support_apk
 331
 332   @property
 333   def test_package(self):
 334     return self._test_package
 335
 336   @property
 337   def test_permissions(self):
 338     return self._test_permissions
 339
 340   @property
 341   def test_runner(self):
 342     return self._test_runner
 343
 344   #override
 345   def TestType(self):
 346     return 'instrumentation'
 347
 348   #override
 349   def SetUp(self):
 350     if self._isolate_delegate:
 351       self._isolate_delegate.Remap(
 352           self._isolate_abs_path, self._isolated_abs_path)
 353       self._isolate_delegate.MoveOutputDeps()
 354       self._data_deps.extend([(constants.ISOLATE_DEPS_DIR, None)])
 355
 356     # TODO(jbudorick): Convert existing tests that depend on the --test-data
 357     # mechanism to isolate, then remove this.
 358     if self._test_data:
 359       for t in self._test_data:
 360         device_rel_path, host_rel_path = t.split(':')
 361         host_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, host_rel_path)
 362         self._data_deps.extend(
 363             [(host_abs_path,
 364               [None, 'chrome', 'test', 'data', device_rel_path])])
 365
 366   def GetDataDependencies(self):
 367     return self._data_deps
 368
 369   def GetTests(self):
 370     pickle_path = '%s-proguard.pickle' % self.test_jar
 371     try:
 372       tests = self._GetTestsFromPickle(pickle_path, self.test_jar)
 373     except self.ProguardPickleException as e:
 374       logging.info('Getting tests from JAR via proguard. (%s)', str(e))
 375       tests = self._GetTestsFromProguard(self.test_jar)
 376       self._SaveTestsToPickle(pickle_path, self.test_jar, tests)
 377     return self._InflateTests(self._FilterTests(tests))
 378
 379   class ProguardPickleException(Exception):
 380     pass
 381
 382   def _GetTestsFromPickle(self, pickle_path, jar_path):
 383     if not os.path.exists(pickle_path):
 384       raise self.ProguardPickleException('%s does not exist.' % pickle_path)
 385     if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
 386       raise self.ProguardPickleException(
 387           '%s newer than %s.' % (jar_path, pickle_path))
 388
 389     with open(pickle_path, 'r') as pickle_file:
 390       pickle_data = pickle.loads(pickle_file.read())
 391     jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
 392
 393     try:
 394       if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION:
 395         raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.')
 396       if pickle_data['JAR_MD5SUM'] != jar_md5:
 397         raise self.ProguardPickleException('JAR file MD5 sum differs.')
 398       return pickle_data['TEST_METHODS']
 399     except TypeError as e:
 400       logging.error(pickle_data)
 401       raise self.ProguardPickleException(str(e))
 402
 403   # pylint: disable=no-self-use
 404   def _GetTestsFromProguard(self, jar_path):
 405     p = proguard.Dump(jar_path)
 406
 407     def is_test_class(c):
 408       return c['class'].endswith('Test')
 409
 410     def is_test_method(m):
 411       return m['method'].startswith('test')
 412
 413     class_lookup = dict((c['class'], c) for c in p['classes'])
 414     def recursive_get_class_annotations(c):
 415       s = c['superclass']
 416       if s in class_lookup:
 417         a = recursive_get_class_annotations(class_lookup[s])
 418       else:
 419         a = {}
 420       a.update(c['annotations'])
 421       return a
 422
 423     def stripped_test_class(c):
 424       return {
 425         'class': c['class'],
 426         'annotations': recursive_get_class_annotations(c),
 427         'methods': [m for m in c['methods'] if is_test_method(m)],
 428       }
 429
 430     return [stripped_test_class(c) for c in p['classes']
 431             if is_test_class(c)]
 432
 433   def _SaveTestsToPickle(self, pickle_path, jar_path, tests):
 434     jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
 435     pickle_data = {
 436       'VERSION': _PICKLE_FORMAT_VERSION,
 437       'JAR_MD5SUM': jar_md5,
 438       'TEST_METHODS': tests,
 439     }
 440     with open(pickle_path, 'w') as pickle_file:
 441       pickle.dump(pickle_data, pickle_file)
 442
 443   def _FilterTests(self, tests):
 444
 445     def gtest_filter(c, m):
 446       t = ['%s.%s' % (c['class'].split('.')[-1], m['method'])]
 447       return (not self._test_filter
 448               or unittest_util.FilterTestNames(t, self._test_filter))
 449
 450     def annotation_filter(all_annotations):
 451       if not self._annotations:
 452         return True
 453       return any_annotation_matches(self._annotations, all_annotations)
 454
 455     def excluded_annotation_filter(all_annotations):
 456       if not self._excluded_annotations:
 457         return True
 458       return not any_annotation_matches(self._excluded_annotations,
 459                                         all_annotations)
 460
 461     def any_annotation_matches(annotations, all_annotations):
 462       return any(
 463           ak in all_annotations and (av is None or av == all_annotations[ak])
 464           for ak, av in annotations.iteritems())
 465
 466     filtered_classes = []
 467     for c in tests:
 468       filtered_methods = []
 469       for m in c['methods']:
 470         # Gtest filtering
 471         if not gtest_filter(c, m):
 472           continue
 473
 474         all_annotations = dict(c['annotations'])
 475         all_annotations.update(m['annotations'])
 476         if (not annotation_filter(all_annotations)
 477             or not excluded_annotation_filter(all_annotations)):
 478           continue
 479
 480         filtered_methods.append(m)
 481
 482       if filtered_methods:
 483         filtered_class = dict(c)
 484         filtered_class['methods'] = filtered_methods
 485         filtered_classes.append(filtered_class)
 486
 487     return filtered_classes
 488
 489   def _InflateTests(self, tests):
 490     inflated_tests = []
 491     for c in tests:
 492       for m in c['methods']:
 493         a = dict(c['annotations'])
 494         a.update(m['annotations'])
 495         inflated_tests.append({
 496             'class': c['class'],
 497             'method': m['method'],
 498             'annotations': a,
 499         })
 500     return inflated_tests
 501
 502   @staticmethod
 503   def GetHttpServerEnvironmentVars():
 504     return {
 505       _EXTRA_ENABLE_HTTP_SERVER: None,
 506     }
 507
 508   def GetDriverEnvironmentVars(
 509       self, test_list=None, test_list_file_path=None):
 510     env = {
 511       _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package,
 512       _EXTRA_DRIVER_TARGET_CLASS: self.test_runner,
 513     }
 514
 515     if test_list:
 516       env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list)
 517
 518     if test_list_file_path:
 519       env[_EXTRA_DRIVER_TEST_LIST_FILE] = (
 520           os.path.basename(test_list_file_path))
 521
 522     return env
 523
 524   @staticmethod
 525   def ParseAmInstrumentRawOutput(raw_output):
 526     return ParseAmInstrumentRawOutput(raw_output)
 527
 528   @staticmethod
 529   def GenerateTestResults(
 530       result_code, result_bundle, statuses, start_ms, duration_ms):
 531     return GenerateTestResults(result_code, result_bundle, statuses,
 532                                start_ms, duration_ms)
 533
 534   #override
 535   def TearDown(self):
 536     if self._isolate_delegate:
 537       self._isolate_delegate.Clear()
 538