build/android/pylib/local/device/local_device_instrumentation_test_run.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import time
   7
   8 from pylib import flag_changer
   9 from pylib.base import base_test_result
  10 from pylib.base import test_run
  11 from pylib.local.device import local_device_test_run
  12
  13
  14 TIMEOUT_ANNOTATIONS = [
  15   ('Manual', 10 * 60 * 60),
  16   ('IntegrationTest', 30 * 60),
  17   ('External', 10 * 60),
  18   ('EnormousTest', 10 * 60),
  19   ('LargeTest', 5 * 60),
  20   ('MediumTest', 3 * 60),
  21   ('SmallTest', 1 * 60),
  22 ]
  23
  24
  25 # TODO(jbudorick): Make this private once the instrumentation test_runner is
  26 # deprecated.
  27 def DidPackageCrashOnDevice(package_name, device):
  28   # Dismiss any error dialogs. Limit the number in case we have an error
  29   # loop or we are failing to dismiss.
  30   for _ in xrange(10):
  31     package = device.old_interface.DismissCrashDialogIfNeeded()
  32     if not package:
  33       return False
  34     # Assume test package convention of ".test" suffix
  35     if package in package_name:
  36       return True
  37   return False
  38
  39
  40 class LocalDeviceInstrumentationTestRun(
  41     local_device_test_run.LocalDeviceTestRun):
  42   def __init__(self, env, test_instance):
  43     super(LocalDeviceInstrumentationTestRun, self).__init__(env, test_instance)
  44     self._flag_changers = {}
  45
  46   def TestPackage(self):
  47     return None
  48
  49   def SetUp(self):
  50     def substitute_external_storage(d, external_storage):
  51       if not d:
  52         return external_storage
  53       elif isinstance(d, list):
  54         return '/'.join(p if p else external_storage for p in d)
  55       else:
  56         return d
  57
  58     def individual_device_set_up(dev, host_device_tuples):
  59       dev.Install(self._test_instance.apk_under_test)
  60       dev.Install(self._test_instance.test_apk)
  61
  62       external_storage = dev.GetExternalStoragePath()
  63       host_device_tuples = [
  64           (h, substitute_external_storage(d, external_storage))
  65           for h, d in host_device_tuples]
  66       logging.info('instrumentation data deps:')
  67       for h, d in host_device_tuples:
  68         logging.info('%r -> %r', h, d)
  69       dev.PushChangedFiles(host_device_tuples)
  70       if self._test_instance.flags:
  71         if not self._test_instance.package_info:
  72           logging.error("Couldn't set flags: no package info")
  73         elif not self._test_instance.package_info.cmdline_file:
  74           logging.error("Couldn't set flags: no cmdline_file")
  75         else:
  76           self._flag_changers[str(dev)] = flag_changer.FlagChanger(
  77               dev, self._test_instance.package_info.cmdline_file)
  78           logging.debug('Attempting to set flags: %r',
  79                         self._test_instance.flags)
  80           self._flag_changers[str(dev)].AddFlags(self._test_instance.flags)
  81
  82     self._env.parallel_devices.pMap(
  83         individual_device_set_up,
  84         self._test_instance.GetDataDependencies())
  85
  86   def TearDown(self):
  87     def individual_device_tear_down(dev):
  88       if str(dev) in self._flag_changers:
  89         self._flag_changers[str(dev)].Restore()
  90
  91     self._env.parallel_devices.pMap(individual_device_tear_down)
  92
  93   #override
  94   def _CreateShards(self, tests):
  95     return tests
  96
  97   #override
  98   def _GetTests(self):
  99     return self._test_instance.GetTests()
 100
 101   #override
 102   def _GetTestName(self, test):
 103     return '%s#%s' % (test['class'], test['method'])
 104
 105   #override
 106   def _RunTest(self, device, test):
 107     test_name = self._GetTestName(test)
 108     logging.info('preparing to run %s: %s' % (test_name, test))
 109
 110     extras = {
 111       'class': test_name,
 112       'org.chromium.chrome.test.ChromeInstrumentationTestRunner'
 113           '.EnableTestHttpServer': '',
 114     }
 115     timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_name)
 116
 117     time_ms = lambda: int(time.time() * 1e3)
 118     start_ms = time_ms()
 119     output = device.StartInstrumentation(
 120         '%s/%s' % (self._test_instance.test_package,
 121                    self._test_instance.test_runner),
 122         raw=True, extras=extras, timeout=timeout, retries=0)
 123     duration_ms = time_ms() - start_ms
 124
 125     # TODO(jbudorick): Make instrumentation tests output a JSON so this
 126     # doesn't have to parse the output.
 127     logging.debug('output from %s:', test_name)
 128     for l in output:
 129       logging.debug('  %s', l)
 130
 131     result_code, result_bundle, statuses = (
 132         self._test_instance.ParseAmInstrumentRawOutput(output))
 133     results = self._test_instance.GenerateTestResults(
 134         result_code, result_bundle, statuses, start_ms, duration_ms)
 135     if DidPackageCrashOnDevice(self._test_instance.test_package, device):
 136       for r in results:
 137         if r.GetType() == base_test_result.ResultType.UNKNOWN:
 138           r.SetType(base_test_result.ResultType.CRASH)
 139     return results
 140
 141   #override
 142   def _ShouldShard(self):
 143     return True
 144
 145   @staticmethod
 146   def _GetTimeoutFromAnnotations(annotations, test_name):
 147     for k, v in TIMEOUT_ANNOTATIONS:
 148       if k in annotations:
 149         timeout = v
 150     else:
 151       logging.warning('Using default 1 minute timeout for %s' % test_name)
 152       timeout = 60
 153
 154     try:
 155       scale = int(annotations.get('TimeoutScale', 1))
 156     except ValueError as e:
 157       logging.warning("Non-integer value of TimeoutScale ignored. (%s)", str(e))
 158       scale = 1
 159     timeout *= scale
 160
 161     return timeout
 162