build/android/pylib/local/device/local_device_instrumentation_test_run.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import re
   7 import time
   8
   9 from devil.android import device_errors
  10 from pylib import flag_changer
  11 from pylib.base import base_test_result
  12 from pylib.local.device import local_device_test_run
  13
  14
  15 TIMEOUT_ANNOTATIONS = [
  16   ('Manual', 10 * 60 * 60),
  17   ('IntegrationTest', 30 * 60),
  18   ('External', 10 * 60),
  19   ('EnormousTest', 10 * 60),
  20   ('LargeTest', 5 * 60),
  21   ('MediumTest', 3 * 60),
  22   ('SmallTest', 1 * 60),
  23 ]
  24
  25
  26 # TODO(jbudorick): Make this private once the instrumentation test_runner is
  27 # deprecated.
  28 def DidPackageCrashOnDevice(package_name, device):
  29   # Dismiss any error dialogs. Limit the number in case we have an error
  30   # loop or we are failing to dismiss.
  31   try:
  32     for _ in xrange(10):
  33       package = device.DismissCrashDialogIfNeeded()
  34       if not package:
  35         return False
  36       # Assume test package convention of ".test" suffix
  37       if package in package_name:
  38         return True
  39   except device_errors.CommandFailedError:
  40     logging.exception('Error while attempting to dismiss crash dialog.')
  41   return False
  42
  43
  44 _CURRENT_FOCUS_CRASH_RE = re.compile(
  45     r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}')
  46
  47
  48 class LocalDeviceInstrumentationTestRun(
  49     local_device_test_run.LocalDeviceTestRun):
  50   def __init__(self, env, test_instance):
  51     super(LocalDeviceInstrumentationTestRun, self).__init__(env, test_instance)
  52     self._flag_changers = {}
  53
  54   def TestPackage(self):
  55     return None
  56
  57   def SetUp(self):
  58     def substitute_external_storage(d, external_storage):
  59       if not d:
  60         return external_storage
  61       elif isinstance(d, list):
  62         return '/'.join(p if p else external_storage for p in d)
  63       else:
  64         return d
  65
  66     def individual_device_set_up(dev, host_device_tuples):
  67       dev.Install(self._test_instance.apk_under_test,
  68                   permissions=self._test_instance.apk_under_test_permissions)
  69       dev.Install(self._test_instance.test_apk,
  70                   permissions=self._test_instance.test_permissions)
  71
  72       external_storage = dev.GetExternalStoragePath()
  73       host_device_tuples = [
  74           (h, substitute_external_storage(d, external_storage))
  75           for h, d in host_device_tuples]
  76       logging.info('instrumentation data deps:')
  77       for h, d in host_device_tuples:
  78         logging.info('%r -> %r', h, d)
  79       dev.PushChangedFiles(host_device_tuples)
  80       if self._test_instance.flags:
  81         if not self._test_instance.package_info:
  82           logging.error("Couldn't set flags: no package info")
  83         elif not self._test_instance.package_info.cmdline_file:
  84           logging.error("Couldn't set flags: no cmdline_file")
  85         else:
  86           self._flag_changers[str(dev)] = flag_changer.FlagChanger(
  87               dev, self._test_instance.package_info.cmdline_file)
  88           logging.debug('Attempting to set flags: %r',
  89                         self._test_instance.flags)
  90           self._flag_changers[str(dev)].AddFlags(self._test_instance.flags)
  91
  92     self._env.parallel_devices.pMap(
  93         individual_device_set_up,
  94         self._test_instance.GetDataDependencies())
  95
  96   def TearDown(self):
  97     def individual_device_tear_down(dev):
  98       if str(dev) in self._flag_changers:
  99         self._flag_changers[str(dev)].Restore()
 100
 101     self._env.parallel_devices.pMap(individual_device_tear_down)
 102
 103   #override
 104   def _CreateShards(self, tests):
 105     return tests
 106
 107   #override
 108   def _GetTests(self):
 109     return self._test_instance.GetTests()
 110
 111   #override
 112   def _GetTestName(self, test):
 113     return '%s#%s' % (test['class'], test['method'])
 114
 115   #override
 116   def _RunTest(self, device, test):
 117     extras = self._test_instance.GetHttpServerEnvironmentVars()
 118
 119     if isinstance(test, list):
 120       if not self._test_instance.driver_apk:
 121         raise Exception('driver_apk does not exist. '
 122                         'Please build it and try again.')
 123
 124       def name_and_timeout(t):
 125         n = self._GetTestName(t)
 126         i = self._GetTimeoutFromAnnotations(t['annotations'], n)
 127         return (n, i)
 128
 129       test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
 130
 131       test_name = ','.join(test_names)
 132       target = '%s/%s' % (
 133           self._test_instance.driver_package,
 134           self._test_instance.driver_name)
 135       extras.update(
 136           self._test_instance.GetDriverEnvironmentVars(
 137               test_list=test_names))
 138       timeout = sum(timeouts)
 139     else:
 140       test_name = self._GetTestName(test)
 141       target = '%s/%s' % (
 142           self._test_instance.test_package, self._test_instance.test_runner)
 143       extras['class'] = test_name
 144       timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_name)
 145
 146     logging.info('preparing to run %s: %s', test_name, test)
 147
 148     time_ms = lambda: int(time.time() * 1e3)
 149     start_ms = time_ms()
 150     output = device.StartInstrumentation(
 151         target, raw=True, extras=extras, timeout=timeout, retries=0)
 152     duration_ms = time_ms() - start_ms
 153
 154     # TODO(jbudorick): Make instrumentation tests output a JSON so this
 155     # doesn't have to parse the output.
 156     logging.debug('output from %s:', test_name)
 157     for l in output:
 158       logging.debug('  %s', l)
 159
 160     result_code, result_bundle, statuses = (
 161         self._test_instance.ParseAmInstrumentRawOutput(output))
 162     results = self._test_instance.GenerateTestResults(
 163         result_code, result_bundle, statuses, start_ms, duration_ms)
 164     if DidPackageCrashOnDevice(self._test_instance.test_package, device):
 165       for r in results:
 166         if r.GetType() == base_test_result.ResultType.UNKNOWN:
 167           r.SetType(base_test_result.ResultType.CRASH)
 168     return results
 169
 170   #override
 171   def _ShouldShard(self):
 172     return True
 173
 174   @staticmethod
 175   def _GetTimeoutFromAnnotations(annotations, test_name):
 176     for k, v in TIMEOUT_ANNOTATIONS:
 177       if k in annotations:
 178         timeout = v
 179         break
 180     else:
 181       logging.warning('Using default 1 minute timeout for %s', test_name)
 182       timeout = 60
 183
 184     try:
 185       scale = int(annotations.get('TimeoutScale', 1))
 186     except ValueError as e:
 187       logging.warning("Non-integer value of TimeoutScale ignored. (%s)", str(e))
 188       scale = 1
 189     timeout *= scale
 190
 191     return timeout
 192