build/android/pylib/local/device/local_device_instrumentation_test_run.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import re
   7 import time
   8
   9 from pylib import flag_changer
  10 from pylib.base import base_test_result
  11 from pylib.base import test_run
  12 from pylib.constants import keyevent
  13 from pylib.device import device_errors
  14 from pylib.local.device import local_device_test_run
  15
  16
  17 TIMEOUT_ANNOTATIONS = [
  18   ('Manual', 10 * 60 * 60),
  19   ('IntegrationTest', 30 * 60),
  20   ('External', 10 * 60),
  21   ('EnormousTest', 10 * 60),
  22   ('LargeTest', 5 * 60),
  23   ('MediumTest', 3 * 60),
  24   ('SmallTest', 1 * 60),
  25 ]
  26
  27
  28 # TODO(jbudorick): Make this private once the instrumentation test_runner is
  29 # deprecated.
  30 def DidPackageCrashOnDevice(package_name, device):
  31   # Dismiss any error dialogs. Limit the number in case we have an error
  32   # loop or we are failing to dismiss.
  33   try:
  34     for _ in xrange(10):
  35       package = _DismissCrashDialog(device)
  36       if not package:
  37         return False
  38       # Assume test package convention of ".test" suffix
  39       if package in package_name:
  40         return True
  41   except device_errors.CommandFailedError:
  42     logging.exception('Error while attempting to dismiss crash dialog.')
  43   return False
  44
  45
  46 _CURRENT_FOCUS_CRASH_RE = re.compile(
  47     r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}')
  48
  49
  50 def _DismissCrashDialog(device):
  51   # TODO(jbudorick): Try to grep the output on the device instead of using
  52   # large_output if/when DeviceUtils exposes a public interface for piped
  53   # shell command handling.
  54   for l in device.RunShellCommand(
  55       ['dumpsys', 'window', 'windows'], check_return=True, large_output=True):
  56     m = re.match(_CURRENT_FOCUS_CRASH_RE, l)
  57     if m:
  58       device.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT)
  59       device.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT)
  60       device.SendKeyEvent(keyevent.KEYCODE_ENTER)
  61       return m.group(2)
  62
  63   return None
  64
  65
  66 class LocalDeviceInstrumentationTestRun(
  67     local_device_test_run.LocalDeviceTestRun):
  68   def __init__(self, env, test_instance):
  69     super(LocalDeviceInstrumentationTestRun, self).__init__(env, test_instance)
  70     self._flag_changers = {}
  71
  72   def TestPackage(self):
  73     return None
  74
  75   def SetUp(self):
  76     def substitute_external_storage(d, external_storage):
  77       if not d:
  78         return external_storage
  79       elif isinstance(d, list):
  80         return '/'.join(p if p else external_storage for p in d)
  81       else:
  82         return d
  83
  84     def individual_device_set_up(dev, host_device_tuples):
  85       dev.Install(self._test_instance.apk_under_test)
  86       dev.Install(self._test_instance.test_apk)
  87
  88       external_storage = dev.GetExternalStoragePath()
  89       host_device_tuples = [
  90           (h, substitute_external_storage(d, external_storage))
  91           for h, d in host_device_tuples]
  92       logging.info('instrumentation data deps:')
  93       for h, d in host_device_tuples:
  94         logging.info('%r -> %r', h, d)
  95       dev.PushChangedFiles(host_device_tuples)
  96       if self._test_instance.flags:
  97         if not self._test_instance.package_info:
  98           logging.error("Couldn't set flags: no package info")
  99         elif not self._test_instance.package_info.cmdline_file:
 100           logging.error("Couldn't set flags: no cmdline_file")
 101         else:
 102           self._flag_changers[str(dev)] = flag_changer.FlagChanger(
 103               dev, self._test_instance.package_info.cmdline_file)
 104           logging.debug('Attempting to set flags: %r',
 105                         self._test_instance.flags)
 106           self._flag_changers[str(dev)].AddFlags(self._test_instance.flags)
 107
 108     self._env.parallel_devices.pMap(
 109         individual_device_set_up,
 110         self._test_instance.GetDataDependencies())
 111
 112   def TearDown(self):
 113     def individual_device_tear_down(dev):
 114       if str(dev) in self._flag_changers:
 115         self._flag_changers[str(dev)].Restore()
 116
 117     self._env.parallel_devices.pMap(individual_device_tear_down)
 118
 119   #override
 120   def _CreateShards(self, tests):
 121     return tests
 122
 123   #override
 124   def _GetTests(self):
 125     return self._test_instance.GetTests()
 126
 127   #override
 128   def _GetTestName(self, test):
 129     return '%s#%s' % (test['class'], test['method'])
 130
 131   #override
 132   def _RunTest(self, device, test):
 133     extras = self._test_instance.GetHttpServerEnvironmentVars()
 134
 135     if isinstance(test, list):
 136       if not self._test_instance.driver_apk:
 137         raise Exception('driver_apk does not exist. '
 138                         'Please build it and try again.')
 139
 140       def name_and_timeout(t):
 141         n = self._GetTestName(t)
 142         i = self._GetTimeoutFromAnnotations(t['annotations'], n)
 143         return (n, i)
 144
 145       test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
 146
 147       test_name = ','.join(test_names)
 148       target = '%s/%s' % (
 149           self._test_instance.driver_package,
 150           self._test_instance.driver_name)
 151       extras.update(
 152           self._test_instance.GetDriverEnvironmentVars(
 153               test_list=test_names))
 154       timeout = sum(timeouts)
 155     else:
 156       test_name = self._GetTestName(test)
 157       target = '%s/%s' % (
 158           self._test_instance.test_package, self._test_instance.test_runner)
 159       extras['class'] = test_name
 160       timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_name)
 161
 162     logging.info('preparing to run %s: %s' % (test_name, test))
 163
 164     time_ms = lambda: int(time.time() * 1e3)
 165     start_ms = time_ms()
 166     output = device.StartInstrumentation(
 167         target, raw=True, extras=extras, timeout=timeout, retries=0)
 168     duration_ms = time_ms() - start_ms
 169
 170     # TODO(jbudorick): Make instrumentation tests output a JSON so this
 171     # doesn't have to parse the output.
 172     logging.debug('output from %s:', test_name)
 173     for l in output:
 174       logging.debug('  %s', l)
 175
 176     result_code, result_bundle, statuses = (
 177         self._test_instance.ParseAmInstrumentRawOutput(output))
 178     results = self._test_instance.GenerateTestResults(
 179         result_code, result_bundle, statuses, start_ms, duration_ms)
 180     if DidPackageCrashOnDevice(self._test_instance.test_package, device):
 181       for r in results:
 182         if r.GetType() == base_test_result.ResultType.UNKNOWN:
 183           r.SetType(base_test_result.ResultType.CRASH)
 184     return results
 185
 186   #override
 187   def _ShouldShard(self):
 188     return True
 189
 190   @staticmethod
 191   def _GetTimeoutFromAnnotations(annotations, test_name):
 192     for k, v in TIMEOUT_ANNOTATIONS:
 193       if k in annotations:
 194         timeout = v
 195     else:
 196       logging.warning('Using default 1 minute timeout for %s' % test_name)
 197       timeout = 60
 198
 199     try:
 200       scale = int(annotations.get('TimeoutScale', 1))
 201     except ValueError as e:
 202       logging.warning("Non-integer value of TimeoutScale ignored. (%s)", str(e))
 203       scale = 1
 204     timeout *= scale
 205
 206     return timeout
 207