build/android/provision_devices.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Provisions Android devices with settings required for bots.
   8
   9 Usage:
  10   ./provision_devices.py [-d <device serial number>]
  11 """
  12
  13 import argparse
  14 import json
  15 import logging
  16 import os
  17 import posixpath
  18 import re
  19 import subprocess
  20 import sys
  21 import time
  22
  23 from pylib import constants
  24 from pylib import device_settings
  25 from pylib.device import battery_utils
  26 from pylib.device import device_blacklist
  27 from pylib.device import device_errors
  28 from pylib.device import device_utils
  29 from pylib.utils import run_tests_helper
  30 from pylib.utils import timeout_retry
  31
  32 sys.path.append(os.path.join(constants.DIR_SOURCE_ROOT,
  33                              'third_party', 'android_testrunner'))
  34 import errors
  35
  36
  37 _SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
  38
  39
  40 class _DEFAULT_TIMEOUTS(object):
  41   # L can take a while to reboot after a wipe.
  42   LOLLIPOP = 600
  43   PRE_LOLLIPOP = 180
  44
  45   HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
  46
  47
  48 class _PHASES(object):
  49   WIPE = 'wipe'
  50   PROPERTIES = 'properties'
  51   FINISH = 'finish'
  52
  53   ALL = [WIPE, PROPERTIES, FINISH]
  54
  55
  56 def ProvisionDevices(options):
  57   devices = device_utils.DeviceUtils.HealthyDevices()
  58   if options.device:
  59     devices = [d for d in devices if d == options.device]
  60     if not devices:
  61       raise device_errors.DeviceUnreachableError(options.device)
  62
  63   parallel_devices = device_utils.DeviceUtils.parallel(devices)
  64   parallel_devices.pMap(ProvisionDevice, options)
  65   if options.auto_reconnect:
  66     _LaunchHostHeartbeat()
  67   blacklist = device_blacklist.ReadBlacklist()
  68   if options.output_device_blacklist:
  69     with open(options.output_device_blacklist, 'w') as f:
  70       json.dump(blacklist, f)
  71   if all(d in blacklist for d in devices):
  72     raise device_errors.NoDevicesError
  73   return 0
  74
  75
  76 def ProvisionDevice(device, options):
  77   if options.reboot_timeout:
  78     reboot_timeout = options.reboot_timeout
  79   elif (device.build_version_sdk >=
  80         constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP):
  81     reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
  82   else:
  83     reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
  84
  85   def should_run_phase(phase_name):
  86     return not options.phases or phase_name in options.phases
  87
  88   def run_phase(phase_func, reboot=True):
  89     try:
  90       device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
  91     except device_errors.CommandTimeoutError:
  92       logging.error('Device did not finish booting. Will try to reboot.')
  93       device.Reboot(timeout=reboot_timeout)
  94     phase_func(device, options)
  95     if reboot:
  96       device.Reboot(False, retries=0)
  97       device.adb.WaitForDevice()
  98
  99   try:
 100     if should_run_phase(_PHASES.WIPE):
 101       run_phase(WipeDevice)
 102
 103     if should_run_phase(_PHASES.PROPERTIES):
 104       run_phase(SetProperties)
 105
 106     if should_run_phase(_PHASES.FINISH):
 107       run_phase(FinishProvisioning, reboot=False)
 108
 109   except (errors.WaitForResponseTimedOutError,
 110           device_errors.CommandTimeoutError):
 111     logging.exception('Timed out waiting for device %s. Adding to blacklist.',
 112                       str(device))
 113     device_blacklist.ExtendBlacklist([str(device)])
 114
 115   except device_errors.CommandFailedError:
 116     logging.exception('Failed to provision device %s. Adding to blacklist.',
 117                       str(device))
 118     device_blacklist.ExtendBlacklist([str(device)])
 119
 120
 121 def WipeDevice(device, options):
 122   """Wipes data from device, keeping only the adb_keys for authorization.
 123
 124   After wiping data on a device that has been authorized, adb can still
 125   communicate with the device, but after reboot the device will need to be
 126   re-authorized because the adb keys file is stored in /data/misc/adb/.
 127   Thus, adb_keys file is rewritten so the device does not need to be
 128   re-authorized.
 129
 130   Arguments:
 131     device: the device to wipe
 132   """
 133   if options.skip_wipe:
 134     return
 135
 136   try:
 137     device.EnableRoot()
 138     device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
 139     if device_authorized:
 140       adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
 141                                  as_root=True).splitlines()
 142     device.RunShellCommand(['wipe', 'data'],
 143                            as_root=True, check_return=True)
 144     device.adb.WaitForDevice()
 145
 146     if device_authorized:
 147       adb_keys_set = set(adb_keys)
 148       for adb_key_file in options.adb_key_files or []:
 149         try:
 150           with open(adb_key_file, 'r') as f:
 151             adb_public_keys = f.readlines()
 152           adb_keys_set.update(adb_public_keys)
 153         except IOError:
 154           logging.warning('Unable to find adb keys file %s.' % adb_key_file)
 155       _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
 156   except device_errors.CommandFailedError:
 157     logging.exception('Possible failure while wiping the device. '
 158                       'Attempting to continue.')
 159
 160
 161 def _WriteAdbKeysFile(device, adb_keys_string):
 162   dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
 163   device.RunShellCommand(['mkdir', '-p', dir_path],
 164                          as_root=True, check_return=True)
 165   device.RunShellCommand(['restorecon', dir_path],
 166                          as_root=True, check_return=True)
 167   device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
 168   device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
 169                          as_root=True, check_return=True)
 170
 171
 172 def SetProperties(device, options):
 173   try:
 174     device.EnableRoot()
 175   except device_errors.CommandFailedError as e:
 176     logging.warning(str(e))
 177
 178   _ConfigureLocalProperties(device, options.enable_java_debug)
 179   device_settings.ConfigureContentSettings(
 180       device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
 181   if options.disable_location:
 182     device_settings.ConfigureContentSettings(
 183         device, device_settings.DISABLE_LOCATION_SETTINGS)
 184   else:
 185     device_settings.ConfigureContentSettings(
 186         device, device_settings.ENABLE_LOCATION_SETTINGS)
 187
 188   if options.disable_mock_location:
 189     device_settings.ConfigureContentSettings(
 190         device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
 191   else:
 192     device_settings.ConfigureContentSettings(
 193         device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
 194
 195   device_settings.SetLockScreenSettings(device)
 196   if options.disable_network:
 197     device_settings.ConfigureContentSettings(
 198         device, device_settings.NETWORK_DISABLED_SETTINGS)
 199
 200   if options.remove_system_webview:
 201     if device.HasRoot():
 202       # This is required, e.g., to replace the system webview on a device.
 203       device.adb.Remount()
 204       device.RunShellCommand(['stop'], check_return=True)
 205       device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
 206                              check_return=True)
 207       device.RunShellCommand(['start'], check_return=True)
 208     else:
 209       logging.warning('Cannot remove system webview from a non-rooted device')
 210
 211
 212 def _ConfigureLocalProperties(device, java_debug=True):
 213   """Set standard readonly testing device properties prior to reboot."""
 214   local_props = [
 215       'persist.sys.usb.config=adb',
 216       'ro.monkey=1',
 217       'ro.test_harness=1',
 218       'ro.audio.silent=1',
 219       'ro.setupwizard.mode=DISABLED',
 220       ]
 221   if java_debug:
 222     local_props.append(
 223         '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
 224     local_props.append('debug.checkjni=1')
 225   try:
 226     device.WriteFile(
 227         constants.DEVICE_LOCAL_PROPERTIES_PATH,
 228         '\n'.join(local_props), as_root=True)
 229     # Android will not respect the local props file if it is world writable.
 230     device.RunShellCommand(
 231         ['chmod', '644', constants.DEVICE_LOCAL_PROPERTIES_PATH],
 232         as_root=True, check_return=True)
 233   except device_errors.CommandFailedError:
 234     logging.exception('Failed to configure local properties.')
 235
 236
 237 def FinishProvisioning(device, options):
 238   if options.min_battery_level is not None:
 239     try:
 240       battery = battery_utils.BatteryUtils(device)
 241       battery.ChargeDeviceToLevel(options.min_battery_level)
 242     except device_errors.CommandFailedError:
 243       logging.exception('Unable to charge device to specified level.')
 244
 245   if options.max_battery_temp is not None:
 246     try:
 247       battery = battery_utils.BatteryUtils(device)
 248       battery.LetBatteryCoolToTemperature(options.max_battery_temp)
 249     except device_errors.CommandFailedError:
 250       logging.exception('Unable to let battery cool to specified temperature.')
 251
 252   device.RunShellCommand(
 253       ['date', '-s', time.strftime('%Y%m%d.%H%M%S', time.gmtime())],
 254       as_root=True, check_return=True)
 255   props = device.RunShellCommand('getprop', check_return=True)
 256   for prop in props:
 257     logging.info('  %s' % prop)
 258   if options.auto_reconnect:
 259     _PushAndLaunchAdbReboot(device, options.target)
 260
 261
 262 def _PushAndLaunchAdbReboot(device, target):
 263   """Pushes and launches the adb_reboot binary on the device.
 264
 265   Arguments:
 266     device: The DeviceUtils instance for the device to which the adb_reboot
 267             binary should be pushed.
 268     target: The build target (example, Debug or Release) which helps in
 269             locating the adb_reboot binary.
 270   """
 271   logging.info('Will push and launch adb_reboot on %s' % str(device))
 272   # Kill if adb_reboot is already running.
 273   device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
 274   # Push adb_reboot
 275   logging.info('  Pushing adb_reboot ...')
 276   adb_reboot = os.path.join(constants.DIR_SOURCE_ROOT,
 277                             'out/%s/adb_reboot' % target)
 278   device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
 279   # Launch adb_reboot
 280   logging.info('  Launching adb_reboot ...')
 281   device.RunShellCommand(
 282       ['/data/local/tmp/adb_reboot'],
 283       check_return=True)
 284
 285
 286 def _LaunchHostHeartbeat():
 287   # Kill if existing host_heartbeat
 288   KillHostHeartbeat()
 289   # Launch a new host_heartbeat
 290   logging.info('Spawning host heartbeat...')
 291   subprocess.Popen([os.path.join(constants.DIR_SOURCE_ROOT,
 292                                  'build/android/host_heartbeat.py')])
 293
 294
 295 def KillHostHeartbeat():
 296   ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
 297   stdout, _ = ps.communicate()
 298   matches = re.findall('\\n.*host_heartbeat.*', stdout)
 299   for match in matches:
 300     logging.info('An instance of host heart beart running... will kill')
 301     pid = re.findall(r'(\S+)', match)[1]
 302     subprocess.call(['kill', str(pid)])
 303
 304
 305 def main():
 306   # Recommended options on perf bots:
 307   # --disable-network
 308   #     TODO(tonyg): We eventually want network on. However, currently radios
 309   #     can cause perfbots to drain faster than they charge.
 310   # --min-battery-level 95
 311   #     Some perf bots run benchmarks with USB charging disabled which leads
 312   #     to gradual draining of the battery. We must wait for a full charge
 313   #     before starting a run in order to keep the devices online.
 314
 315   parser = argparse.ArgumentParser(
 316       description='Provision Android devices with settings required for bots.')
 317   parser.add_argument('-d', '--device', metavar='SERIAL',
 318                       help='the serial number of the device to be provisioned'
 319                       ' (the default is to provision all devices attached)')
 320   parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
 321                       dest='phases',
 322                       help='Phases of provisioning to run. '
 323                            '(If omitted, all phases will be run.)')
 324   parser.add_argument('--skip-wipe', action='store_true', default=False,
 325                       help="don't wipe device data during provisioning")
 326   parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
 327                       help='when wiping the device, max number of seconds to'
 328                       ' wait after each reboot '
 329                       '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
 330   parser.add_argument('--min-battery-level', type=int, metavar='NUM',
 331                       help='wait for the device to reach this minimum battery'
 332                       ' level before trying to continue')
 333   parser.add_argument('--disable-location', action='store_true',
 334                       help='disable Google location services on devices')
 335   parser.add_argument('--disable-mock-location', action='store_true',
 336                       default=False, help='Set ALLOW_MOCK_LOCATION to false')
 337   parser.add_argument('--disable-network', action='store_true',
 338                       help='disable network access on devices')
 339   parser.add_argument('--disable-java-debug', action='store_false',
 340                       dest='enable_java_debug', default=True,
 341                       help='disable Java property asserts and JNI checking')
 342   parser.add_argument('--remove-system-webview', action='store_true',
 343                       help='Remove the system webview from devices.')
 344   parser.add_argument('-t', '--target', default='Debug',
 345                       help='the build target (default: %(default)s)')
 346   parser.add_argument('-r', '--auto-reconnect', action='store_true',
 347                       help='push binary which will reboot the device on adb'
 348                       ' disconnections')
 349   parser.add_argument('--adb-key-files', type=str, nargs='+',
 350                       help='list of adb keys to push to device')
 351   parser.add_argument('-v', '--verbose', action='count', default=1,
 352                       help='Log more information.')
 353   parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
 354                       help='Wait for the battery to have this temp or lower.')
 355   parser.add_argument('--output-device-blacklist',
 356                       help='Json file to output the device blacklist.')
 357   args = parser.parse_args()
 358   constants.SetBuildType(args.target)
 359
 360   run_tests_helper.SetLogLevel(args.verbose)
 361
 362   return ProvisionDevices(args)
 363
 364
 365 if __name__ == '__main__':
 366   sys.exit(main())