build/android/provision_devices.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Provisions Android devices with settings required for bots.
   8
   9 Usage:
  10   ./provision_devices.py [-d <device serial number>]
  11 """
  12
  13 import argparse
  14 import datetime
  15 import json
  16 import logging
  17 import os
  18 import posixpath
  19 import re
  20 import subprocess
  21 import sys
  22 import time
  23
  24 from devil.android import battery_utils
  25 from devil.android import device_blacklist
  26 from devil.android import device_errors
  27 from devil.android import device_utils
  28 from devil.android.sdk import version_codes
  29 from devil.utils import run_tests_helper
  30 from devil.utils import timeout_retry
  31 from pylib import constants
  32 from pylib import device_settings
  33
  34 _SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
  35 _CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
  36 _TOMBSTONE_REGEX = re.compile('tombstone.*')
  37
  38
  39 class _DEFAULT_TIMEOUTS(object):
  40   # L can take a while to reboot after a wipe.
  41   LOLLIPOP = 600
  42   PRE_LOLLIPOP = 180
  43
  44   HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
  45
  46
  47 class _PHASES(object):
  48   WIPE = 'wipe'
  49   PROPERTIES = 'properties'
  50   FINISH = 'finish'
  51
  52   ALL = [WIPE, PROPERTIES, FINISH]
  53
  54
  55 def ProvisionDevices(args):
  56   blacklist = (device_blacklist.Blacklist(args.blacklist_file)
  57                if args.blacklist_file
  58                else None)
  59
  60   devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
  61   if args.device:
  62     devices = [d for d in devices if d == args.device]
  63     if not devices:
  64       raise device_errors.DeviceUnreachableError(args.device)
  65
  66   parallel_devices = device_utils.DeviceUtils.parallel(devices)
  67   parallel_devices.pMap(ProvisionDevice, blacklist, args)
  68   if args.auto_reconnect:
  69     _LaunchHostHeartbeat()
  70   blacklisted_devices = blacklist.Read() if blacklist else []
  71   if args.output_device_blacklist:
  72     with open(args.output_device_blacklist, 'w') as f:
  73       json.dump(blacklisted_devices, f)
  74   if all(d in blacklisted_devices for d in devices):
  75     raise device_errors.NoDevicesError
  76   return 0
  77
  78
  79 def ProvisionDevice(device, blacklist, options):
  80   if options.reboot_timeout:
  81     reboot_timeout = options.reboot_timeout
  82   elif device.build_version_sdk >= version_codes.LOLLIPOP:
  83     reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
  84   else:
  85     reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
  86
  87   def should_run_phase(phase_name):
  88     return not options.phases or phase_name in options.phases
  89
  90   def run_phase(phase_func, reboot=True):
  91     try:
  92       device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
  93     except device_errors.CommandTimeoutError:
  94       logging.error('Device did not finish booting. Will try to reboot.')
  95       device.Reboot(timeout=reboot_timeout)
  96     phase_func(device, options)
  97     if reboot:
  98       device.Reboot(False, retries=0)
  99       device.adb.WaitForDevice()
 100
 101   try:
 102     if should_run_phase(_PHASES.WIPE):
 103       if options.chrome_specific_wipe:
 104         run_phase(WipeChromeData)
 105       else:
 106         run_phase(WipeDevice)
 107
 108     if should_run_phase(_PHASES.PROPERTIES):
 109       run_phase(SetProperties)
 110
 111     if should_run_phase(_PHASES.FINISH):
 112       run_phase(FinishProvisioning, reboot=False)
 113
 114     if options.chrome_specific_wipe:
 115       package = "com.google.android.gms"
 116       version_name = device.GetApplicationVersion(package)
 117       logging.info("Version name for %s is %s", package, version_name)
 118
 119   except device_errors.CommandTimeoutError:
 120     logging.exception('Timed out waiting for device %s. Adding to blacklist.',
 121                       str(device))
 122     blacklist.Extend([str(device)])
 123
 124   except device_errors.CommandFailedError:
 125     logging.exception('Failed to provision device %s. Adding to blacklist.',
 126                       str(device))
 127     blacklist.Extend([str(device)])
 128
 129
 130 def WipeChromeData(device, options):
 131   """Wipes chrome specific data from device
 132   Chrome specific data is:
 133   (1) any dir under /data/data/ whose name matches *chrom*, except
 134       com.android.chrome, which is the chrome stable package
 135   (2) any dir under /data/app/ and /data/app-lib/ whose name matches *chrom*
 136   (3) any files under /data/tombstones/ whose name matches "tombstone*"
 137   (4) /data/local.prop if there is any
 138   (5) /data/local/chrome-command-line if there is any
 139   (6) dir /data/local/.config/ if there is any (this is telemetry related)
 140   (7) dir /data/local/tmp/
 141
 142   Arguments:
 143     device: the device to wipe
 144   """
 145   if options.skip_wipe:
 146     return
 147
 148   try:
 149     device.EnableRoot()
 150     _WipeUnderDirIfMatch(device, '/data/data/', _CHROME_PACKAGE_REGEX,
 151                          constants.PACKAGE_INFO['chrome_stable'].package)
 152     _WipeUnderDirIfMatch(device, '/data/app/', _CHROME_PACKAGE_REGEX)
 153     _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
 154     _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
 155
 156     _WipeFileOrDir(device, '/data/local.prop')
 157     _WipeFileOrDir(device, '/data/local/chrome-command-line')
 158     _WipeFileOrDir(device, '/data/local/.config/')
 159     _WipeFileOrDir(device, '/data/local/tmp/')
 160
 161     device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
 162                            check_return=True)
 163   except device_errors.CommandFailedError:
 164     logging.exception('Possible failure while wiping the device. '
 165                       'Attempting to continue.')
 166
 167
 168 def WipeDevice(device, options):
 169   """Wipes data from device, keeping only the adb_keys for authorization.
 170
 171   After wiping data on a device that has been authorized, adb can still
 172   communicate with the device, but after reboot the device will need to be
 173   re-authorized because the adb keys file is stored in /data/misc/adb/.
 174   Thus, adb_keys file is rewritten so the device does not need to be
 175   re-authorized.
 176
 177   Arguments:
 178     device: the device to wipe
 179   """
 180   if options.skip_wipe:
 181     return
 182
 183   try:
 184     device.EnableRoot()
 185     device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
 186     if device_authorized:
 187       adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
 188                                  as_root=True).splitlines()
 189     device.RunShellCommand(['wipe', 'data'],
 190                            as_root=True, check_return=True)
 191     device.adb.WaitForDevice()
 192
 193     if device_authorized:
 194       adb_keys_set = set(adb_keys)
 195       for adb_key_file in options.adb_key_files or []:
 196         try:
 197           with open(adb_key_file, 'r') as f:
 198             adb_public_keys = f.readlines()
 199           adb_keys_set.update(adb_public_keys)
 200         except IOError:
 201           logging.warning('Unable to find adb keys file %s.', adb_key_file)
 202       _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
 203   except device_errors.CommandFailedError:
 204     logging.exception('Possible failure while wiping the device. '
 205                       'Attempting to continue.')
 206
 207
 208 def _WriteAdbKeysFile(device, adb_keys_string):
 209   dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
 210   device.RunShellCommand(['mkdir', '-p', dir_path],
 211                          as_root=True, check_return=True)
 212   device.RunShellCommand(['restorecon', dir_path],
 213                          as_root=True, check_return=True)
 214   device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
 215   device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
 216                          as_root=True, check_return=True)
 217
 218
 219 def SetProperties(device, options):
 220   try:
 221     device.EnableRoot()
 222   except device_errors.CommandFailedError as e:
 223     logging.warning(str(e))
 224
 225   _ConfigureLocalProperties(device, options.enable_java_debug)
 226   device_settings.ConfigureContentSettings(
 227       device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
 228   if options.disable_location:
 229     device_settings.ConfigureContentSettings(
 230         device, device_settings.DISABLE_LOCATION_SETTINGS)
 231   else:
 232     device_settings.ConfigureContentSettings(
 233         device, device_settings.ENABLE_LOCATION_SETTINGS)
 234
 235   if options.disable_mock_location:
 236     device_settings.ConfigureContentSettings(
 237         device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
 238   else:
 239     device_settings.ConfigureContentSettings(
 240         device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
 241
 242   device_settings.SetLockScreenSettings(device)
 243   if options.disable_network:
 244     device_settings.ConfigureContentSettings(
 245         device, device_settings.NETWORK_DISABLED_SETTINGS)
 246
 247   if options.disable_system_chrome:
 248     # The system chrome version on the device interferes with some tests.
 249     device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
 250                            check_return=True)
 251
 252   if options.remove_system_webview:
 253     if device.HasRoot():
 254       # This is required, e.g., to replace the system webview on a device.
 255       device.adb.Remount()
 256       device.RunShellCommand(['stop'], check_return=True)
 257       device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
 258                              check_return=True)
 259       device.RunShellCommand(['start'], check_return=True)
 260     else:
 261       logging.warning('Cannot remove system webview from a non-rooted device')
 262
 263
 264 def _ConfigureLocalProperties(device, java_debug=True):
 265   """Set standard readonly testing device properties prior to reboot."""
 266   local_props = [
 267       'persist.sys.usb.config=adb',
 268       'ro.monkey=1',
 269       'ro.test_harness=1',
 270       'ro.audio.silent=1',
 271       'ro.setupwizard.mode=DISABLED',
 272       ]
 273   if java_debug:
 274     local_props.append(
 275         '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
 276     local_props.append('debug.checkjni=1')
 277   try:
 278     device.WriteFile(
 279         device.LOCAL_PROPERTIES_PATH,
 280         '\n'.join(local_props), as_root=True)
 281     # Android will not respect the local props file if it is world writable.
 282     device.RunShellCommand(
 283         ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
 284         as_root=True, check_return=True)
 285   except device_errors.CommandFailedError:
 286     logging.exception('Failed to configure local properties.')
 287
 288
 289 def FinishProvisioning(device, options):
 290   if options.min_battery_level is not None:
 291     try:
 292       battery = battery_utils.BatteryUtils(device)
 293       battery.ChargeDeviceToLevel(options.min_battery_level)
 294     except device_errors.CommandFailedError:
 295       logging.exception('Unable to charge device to specified level.')
 296
 297   if options.max_battery_temp is not None:
 298     try:
 299       battery = battery_utils.BatteryUtils(device)
 300       battery.LetBatteryCoolToTemperature(options.max_battery_temp)
 301     except device_errors.CommandFailedError:
 302       logging.exception('Unable to let battery cool to specified temperature.')
 303
 304   def _set_and_verify_date():
 305     if device.build_version_sdk >= version_codes.MARSHMALLOW:
 306       date_format = '%m%d%H%M%Y.%S'
 307       set_date_command = ['date']
 308     else:
 309       date_format = '%Y%m%d.%H%M%S'
 310       set_date_command = ['date', '-s']
 311     strgmtime = time.strftime(date_format, time.gmtime())
 312     set_date_command.append(strgmtime)
 313     device.RunShellCommand(set_date_command, as_root=True, check_return=True)
 314
 315     device_time = device.RunShellCommand(
 316         ['date', '+"%Y%m%d.%H%M%S"'], as_root=True,
 317         single_line=True).replace('"', '')
 318     device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
 319     correct_time = datetime.datetime.strptime(strgmtime, date_format)
 320     tdelta = (correct_time - device_time).seconds
 321     if tdelta <= 1:
 322       logging.info('Date/time successfully set on %s', device)
 323       return True
 324     else:
 325       logging.error('Date mismatch. Device: %s Correct: %s',
 326                     device_time.isoformat(), correct_time.isoformat())
 327       return False
 328
 329   # Sometimes the date is not set correctly on the devices. Retry on failure.
 330   if not timeout_retry.WaitFor(_set_and_verify_date, wait_period=1,
 331                                max_tries=2):
 332     raise device_errors.CommandFailedError(
 333         'Failed to set date & time.', device_serial=str(device))
 334
 335   props = device.RunShellCommand('getprop', check_return=True)
 336   for prop in props:
 337     logging.info('  %s', prop)
 338   if options.auto_reconnect:
 339     _PushAndLaunchAdbReboot(device, options.target)
 340
 341
 342 def _WipeUnderDirIfMatch(device, path, pattern, app_to_keep=None):
 343   ls_result = device.Ls(path)
 344   for (content, _) in ls_result:
 345     if pattern.match(content):
 346       if content != app_to_keep:
 347         _WipeFileOrDir(device, path + content)
 348
 349
 350 def _WipeFileOrDir(device, path):
 351   if device.PathExists(path):
 352     device.RunShellCommand(['rm', '-rf', path], check_return=True)
 353
 354
 355 def _PushAndLaunchAdbReboot(device, target):
 356   """Pushes and launches the adb_reboot binary on the device.
 357
 358   Arguments:
 359     device: The DeviceUtils instance for the device to which the adb_reboot
 360             binary should be pushed.
 361     target: The build target (example, Debug or Release) which helps in
 362             locating the adb_reboot binary.
 363   """
 364   logging.info('Will push and launch adb_reboot on %s', str(device))
 365   # Kill if adb_reboot is already running.
 366   device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
 367   # Push adb_reboot
 368   logging.info('  Pushing adb_reboot ...')
 369   adb_reboot = os.path.join(constants.DIR_SOURCE_ROOT,
 370                             'out/%s/adb_reboot' % target)
 371   device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
 372   # Launch adb_reboot
 373   logging.info('  Launching adb_reboot ...')
 374   device.RunShellCommand(
 375       ['/data/local/tmp/adb_reboot'],
 376       check_return=True)
 377
 378
 379 def _LaunchHostHeartbeat():
 380   # Kill if existing host_heartbeat
 381   KillHostHeartbeat()
 382   # Launch a new host_heartbeat
 383   logging.info('Spawning host heartbeat...')
 384   subprocess.Popen([os.path.join(constants.DIR_SOURCE_ROOT,
 385                                  'build/android/host_heartbeat.py')])
 386
 387
 388 def KillHostHeartbeat():
 389   ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
 390   stdout, _ = ps.communicate()
 391   matches = re.findall('\\n.*host_heartbeat.*', stdout)
 392   for match in matches:
 393     logging.info('An instance of host heart beart running... will kill')
 394     pid = re.findall(r'(\S+)', match)[1]
 395     subprocess.call(['kill', str(pid)])
 396
 397
 398 def main():
 399   # Recommended options on perf bots:
 400   # --disable-network
 401   #     TODO(tonyg): We eventually want network on. However, currently radios
 402   #     can cause perfbots to drain faster than they charge.
 403   # --min-battery-level 95
 404   #     Some perf bots run benchmarks with USB charging disabled which leads
 405   #     to gradual draining of the battery. We must wait for a full charge
 406   #     before starting a run in order to keep the devices online.
 407
 408   parser = argparse.ArgumentParser(
 409       description='Provision Android devices with settings required for bots.')
 410   parser.add_argument('-d', '--device', metavar='SERIAL',
 411                       help='the serial number of the device to be provisioned'
 412                       ' (the default is to provision all devices attached)')
 413   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 414   parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
 415                       dest='phases',
 416                       help='Phases of provisioning to run. '
 417                            '(If omitted, all phases will be run.)')
 418   parser.add_argument('--skip-wipe', action='store_true', default=False,
 419                       help="don't wipe device data during provisioning")
 420   parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
 421                       help='when wiping the device, max number of seconds to'
 422                       ' wait after each reboot '
 423                       '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
 424   parser.add_argument('--min-battery-level', type=int, metavar='NUM',
 425                       help='wait for the device to reach this minimum battery'
 426                       ' level before trying to continue')
 427   parser.add_argument('--disable-location', action='store_true',
 428                       help='disable Google location services on devices')
 429   parser.add_argument('--disable-mock-location', action='store_true',
 430                       default=False, help='Set ALLOW_MOCK_LOCATION to false')
 431   parser.add_argument('--disable-network', action='store_true',
 432                       help='disable network access on devices')
 433   parser.add_argument('--disable-java-debug', action='store_false',
 434                       dest='enable_java_debug', default=True,
 435                       help='disable Java property asserts and JNI checking')
 436   parser.add_argument('--disable-system-chrome', action='store_true',
 437                       help='Disable the system chrome from devices.')
 438   parser.add_argument('--remove-system-webview', action='store_true',
 439                       help='Remove the system webview from devices.')
 440   parser.add_argument('-t', '--target', default='Debug',
 441                       help='the build target (default: %(default)s)')
 442   parser.add_argument('-r', '--auto-reconnect', action='store_true',
 443                       help='push binary which will reboot the device on adb'
 444                       ' disconnections')
 445   parser.add_argument('--adb-key-files', type=str, nargs='+',
 446                       help='list of adb keys to push to device')
 447   parser.add_argument('-v', '--verbose', action='count', default=1,
 448                       help='Log more information.')
 449   parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
 450                       help='Wait for the battery to have this temp or lower.')
 451   parser.add_argument('--output-device-blacklist',
 452                       help='Json file to output the device blacklist.')
 453   parser.add_argument('--chrome-specific-wipe', action='store_true',
 454                       help='only wipe chrome specific data during provisioning')
 455   args = parser.parse_args()
 456   constants.SetBuildType(args.target)
 457
 458   run_tests_helper.SetLogLevel(args.verbose)
 459
 460   return ProvisionDevices(args)
 461
 462
 463 if __name__ == '__main__':
 464   sys.exit(main())