build/android/pylib/remote/device/remote_device_test_run.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Run specific test on specific environment."""
   6
   7 import json
   8 import logging
   9 import os
  10 import sys
  11 import tempfile
  12 import time
  13 import zipfile
  14
  15 from pylib import constants
  16 from pylib.base import test_run
  17 from pylib.remote.device import appurify_constants
  18 from pylib.remote.device import appurify_sanitized
  19 from pylib.remote.device import remote_device_helper
  20 from pylib.utils import zip_utils
  21
  22 class RemoteDeviceTestRun(test_run.TestRun):
  23   """Run tests on a remote device."""
  24
  25   _TEST_RUN_KEY = 'test_run'
  26   _TEST_RUN_ID_KEY = 'test_run_id'
  27
  28   WAIT_TIME = 5
  29   COMPLETE = 'complete'
  30   HEARTBEAT_INTERVAL = 300
  31
  32   def __init__(self, env, test_instance):
  33     """Constructor.
  34
  35     Args:
  36       env: Environment the tests will run in.
  37       test_instance: The test that will be run.
  38     """
  39     super(RemoteDeviceTestRun, self).__init__(env, test_instance)
  40     self._env = env
  41     self._test_instance = test_instance
  42     self._app_id = ''
  43     self._test_id = ''
  44     self._results = ''
  45     self._test_run_id = ''
  46
  47   #override
  48   def SetUp(self):
  49     """Set up a test run."""
  50     if self._env.trigger:
  51       self._TriggerSetUp()
  52     elif self._env.collect:
  53       assert isinstance(self._env.collect, basestring), (
  54                         'File for storing test_run_id must be a string.')
  55       with open(self._env.collect, 'r') as persisted_data_file:
  56         persisted_data = json.loads(persisted_data_file.read())
  57         self._env.LoadFrom(persisted_data)
  58         self.LoadFrom(persisted_data)
  59
  60   def _TriggerSetUp(self):
  61     """Set up the triggering of a test run."""
  62     raise NotImplementedError
  63
  64   #override
  65   def RunTests(self):
  66     """Run the test."""
  67     if self._env.trigger:
  68       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
  69                                               logging.WARNING):
  70         test_start_res = appurify_sanitized.api.tests_run(
  71             self._env.token, self._env.device_type_id, self._app_id,
  72             self._test_id)
  73       remote_device_helper.TestHttpResponse(
  74         test_start_res, 'Unable to run test.')
  75       self._test_run_id = test_start_res.json()['response']['test_run_id']
  76       logging.info('Test run id: %s' % self._test_run_id)
  77
  78     if self._env.collect:
  79       current_status = ''
  80       timeout_counter = 0
  81       heartbeat_counter = 0
  82       while self._GetTestStatus(self._test_run_id) != self.COMPLETE:
  83         if self._results['detailed_status'] != current_status:
  84           logging.info('Test status: %s', self._results['detailed_status'])
  85           current_status = self._results['detailed_status']
  86           timeout_counter = 0
  87           heartbeat_counter = 0
  88         if heartbeat_counter > self.HEARTBEAT_INTERVAL:
  89           logging.info('Test status: %s', self._results['detailed_status'])
  90           heartbeat_counter = 0
  91
  92         timeout = self._env.timeouts.get(
  93             current_status, self._env.timeouts['unknown'])
  94         if timeout_counter > timeout:
  95           raise remote_device_helper.RemoteDeviceError(
  96               'Timeout while in %s state for %s seconds'
  97               % (current_status, timeout),
  98               is_infra_error=True)
  99         time.sleep(self.WAIT_TIME)
 100         timeout_counter += self.WAIT_TIME
 101         heartbeat_counter += self.WAIT_TIME
 102       self._DownloadTestResults(self._env.results_path)
 103
 104       if self._results['results']['exception']:
 105         raise remote_device_helper.RemoteDeviceError(
 106             self._results['results']['exception'], is_infra_error=True)
 107
 108       return self._ParseTestResults()
 109
 110   #override
 111   def TearDown(self):
 112     """Tear down the test run."""
 113     if self._env.collect:
 114       self._CollectTearDown()
 115     elif self._env.trigger:
 116       assert isinstance(self._env.trigger, basestring), (
 117                         'File for storing test_run_id must be a string.')
 118       with open(self._env.trigger, 'w') as persisted_data_file:
 119         persisted_data = {}
 120         self.DumpTo(persisted_data)
 121         self._env.DumpTo(persisted_data)
 122         persisted_data_file.write(json.dumps(persisted_data))
 123
 124   def _CollectTearDown(self):
 125     if self._GetTestStatus(self._test_run_id) != self.COMPLETE:
 126       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 127                                               logging.WARNING):
 128         test_abort_res = appurify_sanitized.api.tests_abort(
 129             self._env.token, self._test_run_id, reason='Test runner exiting.')
 130       remote_device_helper.TestHttpResponse(test_abort_res,
 131                                             'Unable to abort test.')
 132
 133   def __enter__(self):
 134     """Set up the test run when used as a context manager."""
 135     self.SetUp()
 136     return self
 137
 138   def __exit__(self, exc_type, exc_val, exc_tb):
 139     """Tear down the test run when used as a context manager."""
 140     self.TearDown()
 141
 142   def DumpTo(self, persisted_data):
 143     test_run_data = {
 144       self._TEST_RUN_ID_KEY: self._test_run_id,
 145     }
 146     persisted_data[self._TEST_RUN_KEY] = test_run_data
 147
 148   def LoadFrom(self, persisted_data):
 149     test_run_data = persisted_data[self._TEST_RUN_KEY]
 150     self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY]
 151
 152   def _ParseTestResults(self):
 153     raise NotImplementedError
 154
 155   def _GetTestByName(self, test_name):
 156     """Gets test_id for specific test.
 157
 158     Args:
 159       test_name: Test to find the ID of.
 160     """
 161     with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 162                                             logging.WARNING):
 163       test_list_res = appurify_sanitized.api.tests_list(self._env.token)
 164     remote_device_helper.TestHttpResponse(test_list_res,
 165                                           'Unable to get tests list.')
 166     for test in test_list_res.json()['response']:
 167       if test['test_type'] == test_name:
 168         return test['test_id']
 169     raise remote_device_helper.RemoteDeviceError(
 170         'No test found with name %s' % (test_name))
 171
 172   def _DownloadTestResults(self, results_path):
 173     """Download the test results from remote device service.
 174
 175     Args:
 176       results_path: Path to download appurify results zipfile.
 177     """
 178     if results_path:
 179       logging.info('Downloading results to %s.' % results_path)
 180       if not os.path.exists(os.path.dirname(results_path)):
 181         os.makedirs(os.path.dirname(results_path))
 182       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 183                                               logging.WARNING):
 184         appurify_sanitized.utils.wget(self._results['results']['url'],
 185                                       results_path)
 186
 187   def _GetTestStatus(self, test_run_id):
 188     """Checks the state of the test, and sets self._results
 189
 190     Args:
 191       test_run_id: Id of test on on remote service.
 192     """
 193
 194     with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 195                                             logging.WARNING):
 196       test_check_res = appurify_sanitized.api.tests_check_result(
 197           self._env.token, test_run_id)
 198     remote_device_helper.TestHttpResponse(test_check_res,
 199                                           'Unable to get test status.')
 200     self._results = test_check_res.json()['response']
 201     return self._results['status']
 202
 203   def _AmInstrumentTestSetup(self, app_path, test_path, runner_package,
 204                              environment_variables, extra_apks=None):
 205     config = {'runner': runner_package}
 206     if environment_variables:
 207       config['environment_vars'] = ','.join(
 208           '%s=%s' % (k, v) for k, v in environment_variables.iteritems())
 209
 210     self._app_id = self._UploadAppToDevice(app_path)
 211
 212     data_deps = self._test_instance.GetDataDependencies()
 213     if data_deps:
 214       with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps:
 215         sdcard_files = []
 216         additional_apks = []
 217         host_test = os.path.basename(test_path)
 218         with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file:
 219           zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED)
 220           for h, _ in data_deps:
 221             if os.path.isdir(h):
 222               zip_utils.WriteToZipFile(zip_file, h, '.')
 223               sdcard_files.extend(os.listdir(h))
 224             else:
 225               zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h))
 226               sdcard_files.append(os.path.basename(h))
 227           for a in extra_apks or ():
 228             zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a));
 229             additional_apks.append(os.path.basename(a))
 230
 231         config['sdcard_files'] = ','.join(sdcard_files)
 232         config['host_test'] = host_test
 233         if additional_apks:
 234           config['additional_apks'] = ','.join(additional_apks)
 235         self._test_id = self._UploadTestToDevice(
 236             'robotium', test_with_deps.name, app_id=self._app_id)
 237     else:
 238       self._test_id = self._UploadTestToDevice('robotium', test_path)
 239
 240     logging.info('Setting config: %s' % config)
 241     appurify_configs = {}
 242     if self._env.network_config:
 243       appurify_configs['network'] = self._env.network_config
 244     self._SetTestConfig('robotium', config, **appurify_configs)
 245
 246   def _UploadAppToDevice(self, app_path):
 247     """Upload app to device."""
 248     logging.info('Uploading %s to remote service as %s.', app_path,
 249                  self._test_instance.suite)
 250     with open(app_path, 'rb') as apk_src:
 251       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 252                                               logging.WARNING):
 253         upload_results = appurify_sanitized.api.apps_upload(
 254             self._env.token, apk_src, 'raw', name=self._test_instance.suite)
 255       remote_device_helper.TestHttpResponse(
 256           upload_results, 'Unable to upload %s.' % app_path)
 257       return upload_results.json()['response']['app_id']
 258
 259   def _UploadTestToDevice(self, test_type, test_path, app_id=None):
 260     """Upload test to device
 261     Args:
 262       test_type: Type of test that is being uploaded. Ex. uirobot, gtest..
 263     """
 264     logging.info('Uploading %s to remote service.' % test_path)
 265     with open(test_path, 'rb') as test_src:
 266       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 267                                               logging.WARNING):
 268         upload_results = appurify_sanitized.api.tests_upload(
 269             self._env.token, test_src, 'raw', test_type, app_id=app_id)
 270       remote_device_helper.TestHttpResponse(upload_results,
 271           'Unable to upload %s.' % test_path)
 272       return upload_results.json()['response']['test_id']
 273
 274   def _SetTestConfig(self, runner_type, runner_configs,
 275                      network=appurify_constants.NETWORK.WIFI_1_BAR,
 276                      pcap=0, profiler=0, videocapture=0):
 277     """Generates and uploads config file for test.
 278     Args:
 279       runner_configs: Configs specific to the runner you are using.
 280       network: Config to specify the network environment the devices running
 281           the tests will be in.
 282       pcap: Option to set the recording the of network traffic from the device.
 283       profiler: Option to set the recording of CPU, memory, and network
 284           transfer usage in the tests.
 285       videocapture: Option to set video capture during the tests.
 286
 287     """
 288     logging.info('Generating config file for test.')
 289     with tempfile.TemporaryFile() as config:
 290       config_data = [
 291           '[appurify]',
 292           'network=%s' % network,
 293           'pcap=%s' % pcap,
 294           'profiler=%s' % profiler,
 295           'videocapture=%s' % videocapture,
 296           '[%s]' % runner_type
 297       ]
 298       config_data.extend(
 299           '%s=%s' % (k, v) for k, v in runner_configs.iteritems())
 300       config.write(''.join('%s\n' % l for l in config_data))
 301       config.flush()
 302       config.seek(0)
 303       with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
 304                                               logging.WARNING):
 305         config_response = appurify_sanitized.api.config_upload(
 306             self._env.token, config, self._test_id)
 307       remote_device_helper.TestHttpResponse(
 308           config_response, 'Unable to upload test config.')