build/android/provision_devices.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Provisions Android devices with settings required for bots.
   8
   9 Usage:
  10   ./provision_devices.py [-d <device serial number>]
  11 """
  12
  13 import argparse
  14 import datetime
  15 import json
  16 import logging
  17 import os
  18 import posixpath
  19 import re
  20 import subprocess
  21 import sys
  22 import time
  23
  24 from devil.android import battery_utils
  25 from devil.android import device_blacklist
  26 from devil.android import device_errors
  27 from devil.android import device_utils
  28 from devil.android.sdk import version_codes
  29 from devil.utils import run_tests_helper
  30 from devil.utils import timeout_retry
  31 from pylib import constants
  32 from pylib import device_settings
  33
  34 _SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
  35 _CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
  36 _TOMBSTONE_REGEX = re.compile('tombstone.*')
  37
  38
  39 class _DEFAULT_TIMEOUTS(object):
  40   # L can take a while to reboot after a wipe.
  41   LOLLIPOP = 600
  42   PRE_LOLLIPOP = 180
  43
  44   HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
  45
  46
  47 class _PHASES(object):
  48   WIPE = 'wipe'
  49   PROPERTIES = 'properties'
  50   FINISH = 'finish'
  51
  52   ALL = [WIPE, PROPERTIES, FINISH]
  53
  54
  55 def ProvisionDevices(args):
  56   blacklist = (device_blacklist.Blacklist(args.blacklist_file)
  57                if args.blacklist_file
  58                else None)
  59
  60   devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
  61   if args.device:
  62     devices = [d for d in devices if d == args.device]
  63     if not devices:
  64       raise device_errors.DeviceUnreachableError(args.device)
  65
  66   parallel_devices = device_utils.DeviceUtils.parallel(devices)
  67   parallel_devices.pMap(ProvisionDevice, blacklist, args)
  68   if args.auto_reconnect:
  69     _LaunchHostHeartbeat()
  70   blacklisted_devices = blacklist.Read() if blacklist else []
  71   if args.output_device_blacklist:
  72     with open(args.output_device_blacklist, 'w') as f:
  73       json.dump(blacklisted_devices, f)
  74   if all(d in blacklisted_devices for d in devices):
  75     raise device_errors.NoDevicesError
  76   return 0
  77
  78
  79 def ProvisionDevice(device, blacklist, options):
  80   if options.reboot_timeout:
  81     reboot_timeout = options.reboot_timeout
  82   elif device.build_version_sdk >= version_codes.LOLLIPOP:
  83     reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
  84   else:
  85     reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
  86
  87   def should_run_phase(phase_name):
  88     return not options.phases or phase_name in options.phases
  89
  90   def run_phase(phase_func, reboot=True):
  91     try:
  92       device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
  93     except device_errors.CommandTimeoutError:
  94       logging.error('Device did not finish booting. Will try to reboot.')
  95       device.Reboot(timeout=reboot_timeout)
  96     phase_func(device, options)
  97     if reboot:
  98       device.Reboot(False, retries=0)
  99       device.adb.WaitForDevice()
 100
 101   try:
 102     if should_run_phase(_PHASES.WIPE):
 103       if options.chrome_specific_wipe:
 104         run_phase(WipeChromeData)
 105       else:
 106         run_phase(WipeDevice)
 107
 108     if should_run_phase(_PHASES.PROPERTIES):
 109       run_phase(SetProperties)
 110
 111     if should_run_phase(_PHASES.FINISH):
 112       run_phase(FinishProvisioning, reboot=False)
 113
 114   except device_errors.CommandTimeoutError:
 115     logging.exception('Timed out waiting for device %s. Adding to blacklist.',
 116                       str(device))
 117     blacklist.Extend([str(device)])
 118
 119   except device_errors.CommandFailedError:
 120     logging.exception('Failed to provision device %s. Adding to blacklist.',
 121                       str(device))
 122     blacklist.Extend([str(device)])
 123
 124
 125 def WipeChromeData(device, options):
 126   """Wipes chrome specific data from device
 127   Chrome specific data is:
 128   (1) any dir under /data/data/ whose name matches *chrom*, except
 129       com.android.chrome, which is the chrome stable package
 130   (2) any dir under /data/app/ and /data/app-lib/ whose name matches *chrom*
 131   (3) any files under /data/tombstones/ whose name matches "tombstone*"
 132   (4) /data/local.prop if there is any
 133   (5) /data/local/chrome-command-line if there is any
 134   (6) dir /data/local/.config/ if there is any (this is telemetry related)
 135   (7) dir /data/local/tmp/
 136
 137   Arguments:
 138     device: the device to wipe
 139   """
 140   if options.skip_wipe:
 141     return
 142
 143   try:
 144     device.EnableRoot()
 145     _WipeUnderDirIfMatch(device, '/data/data/', _CHROME_PACKAGE_REGEX,
 146                          constants.PACKAGE_INFO['chrome_stable'].package)
 147     _WipeUnderDirIfMatch(device, '/data/app/', _CHROME_PACKAGE_REGEX)
 148     _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
 149     _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
 150
 151     _WipeFileOrDir(device, '/data/local.prop')
 152     _WipeFileOrDir(device, '/data/local/chrome-command-line')
 153     _WipeFileOrDir(device, '/data/local/.config/')
 154     _WipeFileOrDir(device, '/data/local/tmp/')
 155
 156     device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
 157                            check_return=True)
 158   except device_errors.CommandFailedError:
 159     logging.exception('Possible failure while wiping the device. '
 160                       'Attempting to continue.')
 161
 162
 163 def WipeDevice(device, options):
 164   """Wipes data from device, keeping only the adb_keys for authorization.
 165
 166   After wiping data on a device that has been authorized, adb can still
 167   communicate with the device, but after reboot the device will need to be
 168   re-authorized because the adb keys file is stored in /data/misc/adb/.
 169   Thus, adb_keys file is rewritten so the device does not need to be
 170   re-authorized.
 171
 172   Arguments:
 173     device: the device to wipe
 174   """
 175   if options.skip_wipe:
 176     return
 177
 178   try:
 179     device.EnableRoot()
 180     device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
 181     if device_authorized:
 182       adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
 183                                  as_root=True).splitlines()
 184     device.RunShellCommand(['wipe', 'data'],
 185                            as_root=True, check_return=True)
 186     device.adb.WaitForDevice()
 187
 188     if device_authorized:
 189       adb_keys_set = set(adb_keys)
 190       for adb_key_file in options.adb_key_files or []:
 191         try:
 192           with open(adb_key_file, 'r') as f:
 193             adb_public_keys = f.readlines()
 194           adb_keys_set.update(adb_public_keys)
 195         except IOError:
 196           logging.warning('Unable to find adb keys file %s.', adb_key_file)
 197       _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
 198   except device_errors.CommandFailedError:
 199     logging.exception('Possible failure while wiping the device. '
 200                       'Attempting to continue.')
 201
 202
 203 def _WriteAdbKeysFile(device, adb_keys_string):
 204   dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
 205   device.RunShellCommand(['mkdir', '-p', dir_path],
 206                          as_root=True, check_return=True)
 207   device.RunShellCommand(['restorecon', dir_path],
 208                          as_root=True, check_return=True)
 209   device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
 210   device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
 211                          as_root=True, check_return=True)
 212
 213
 214 def SetProperties(device, options):
 215   try:
 216     device.EnableRoot()
 217   except device_errors.CommandFailedError as e:
 218     logging.warning(str(e))
 219
 220   _ConfigureLocalProperties(device, options.enable_java_debug)
 221   device_settings.ConfigureContentSettings(
 222       device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
 223   if options.disable_location:
 224     device_settings.ConfigureContentSettings(
 225         device, device_settings.DISABLE_LOCATION_SETTINGS)
 226   else:
 227     device_settings.ConfigureContentSettings(
 228         device, device_settings.ENABLE_LOCATION_SETTINGS)
 229
 230   if options.disable_mock_location:
 231     device_settings.ConfigureContentSettings(
 232         device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
 233   else:
 234     device_settings.ConfigureContentSettings(
 235         device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
 236
 237   device_settings.SetLockScreenSettings(device)
 238   if options.disable_network:
 239     device_settings.ConfigureContentSettings(
 240         device, device_settings.NETWORK_DISABLED_SETTINGS)
 241
 242   if options.disable_system_chrome:
 243     # The system chrome version on the device interferes with some tests.
 244     device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
 245                            check_return=True)
 246
 247   if options.remove_system_webview:
 248     if device.HasRoot():
 249       # This is required, e.g., to replace the system webview on a device.
 250       device.adb.Remount()
 251       device.RunShellCommand(['stop'], check_return=True)
 252       device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
 253                              check_return=True)
 254       device.RunShellCommand(['start'], check_return=True)
 255     else:
 256       logging.warning('Cannot remove system webview from a non-rooted device')
 257
 258
 259 def _ConfigureLocalProperties(device, java_debug=True):
 260   """Set standard readonly testing device properties prior to reboot."""
 261   local_props = [
 262       'persist.sys.usb.config=adb',
 263       'ro.monkey=1',
 264       'ro.test_harness=1',
 265       'ro.audio.silent=1',
 266       'ro.setupwizard.mode=DISABLED',
 267       ]
 268   if java_debug:
 269     local_props.append(
 270         '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
 271     local_props.append('debug.checkjni=1')
 272   try:
 273     device.WriteFile(
 274         device.LOCAL_PROPERTIES_PATH,
 275         '\n'.join(local_props), as_root=True)
 276     # Android will not respect the local props file if it is world writable.
 277     device.RunShellCommand(
 278         ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
 279         as_root=True, check_return=True)
 280   except device_errors.CommandFailedError:
 281     logging.exception('Failed to configure local properties.')
 282
 283
 284 def FinishProvisioning(device, options):
 285   if options.min_battery_level is not None:
 286     try:
 287       battery = battery_utils.BatteryUtils(device)
 288       battery.ChargeDeviceToLevel(options.min_battery_level)
 289     except device_errors.CommandFailedError:
 290       logging.exception('Unable to charge device to specified level.')
 291
 292   if options.max_battery_temp is not None:
 293     try:
 294       battery = battery_utils.BatteryUtils(device)
 295       battery.LetBatteryCoolToTemperature(options.max_battery_temp)
 296     except device_errors.CommandFailedError:
 297       logging.exception('Unable to let battery cool to specified temperature.')
 298
 299   def _set_and_verify_date():
 300     if device.build_version_sdk >= version_codes.MARSHMALLOW:
 301       date_format = '%m%d%H%M%Y.%S'
 302       set_date_command = ['date']
 303     else:
 304       date_format = '%Y%m%d.%H%M%S'
 305       set_date_command = ['date', '-s']
 306     strgmtime = time.strftime(date_format, time.gmtime())
 307     set_date_command.append(strgmtime)
 308     device.RunShellCommand(set_date_command, as_root=True, check_return=True)
 309
 310     device_time = device.RunShellCommand(
 311         ['date', '+"%Y%m%d.%H%M%S"'], as_root=True,
 312         single_line=True).replace('"', '')
 313     device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
 314     correct_time = datetime.datetime.strptime(strgmtime, date_format)
 315     tdelta = (correct_time - device_time).seconds
 316     if tdelta <= 1:
 317       logging.info('Date/time successfully set on %s', device)
 318       return True
 319     else:
 320       logging.error('Date mismatch. Device: %s Correct: %s',
 321                     device_time.isoformat(), correct_time.isoformat())
 322       return False
 323
 324   # Sometimes the date is not set correctly on the devices. Retry on failure.
 325   if not timeout_retry.WaitFor(_set_and_verify_date, wait_period=1,
 326                                max_tries=2):
 327     raise device_errors.CommandFailedError(
 328         'Failed to set date & time.', device_serial=str(device))
 329
 330   props = device.RunShellCommand('getprop', check_return=True)
 331   for prop in props:
 332     logging.info('  %s', prop)
 333   if options.auto_reconnect:
 334     _PushAndLaunchAdbReboot(device, options.target)
 335
 336
 337 def _WipeUnderDirIfMatch(device, path, pattern, app_to_keep=None):
 338   ls_result = device.Ls(path)
 339   for (content, _) in ls_result:
 340     if pattern.match(content):
 341       if content != app_to_keep:
 342         _WipeFileOrDir(device, path + content)
 343
 344
 345 def _WipeFileOrDir(device, path):
 346   if device.PathExists(path):
 347     device.RunShellCommand(['rm', '-rf', path], check_return=True)
 348
 349
 350 def _PushAndLaunchAdbReboot(device, target):
 351   """Pushes and launches the adb_reboot binary on the device.
 352
 353   Arguments:
 354     device: The DeviceUtils instance for the device to which the adb_reboot
 355             binary should be pushed.
 356     target: The build target (example, Debug or Release) which helps in
 357             locating the adb_reboot binary.
 358   """
 359   logging.info('Will push and launch adb_reboot on %s', str(device))
 360   # Kill if adb_reboot is already running.
 361   device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
 362   # Push adb_reboot
 363   logging.info('  Pushing adb_reboot ...')
 364   adb_reboot = os.path.join(constants.DIR_SOURCE_ROOT,
 365                             'out/%s/adb_reboot' % target)
 366   device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
 367   # Launch adb_reboot
 368   logging.info('  Launching adb_reboot ...')
 369   device.RunShellCommand(
 370       ['/data/local/tmp/adb_reboot'],
 371       check_return=True)
 372
 373
 374 def _LaunchHostHeartbeat():
 375   # Kill if existing host_heartbeat
 376   KillHostHeartbeat()
 377   # Launch a new host_heartbeat
 378   logging.info('Spawning host heartbeat...')
 379   subprocess.Popen([os.path.join(constants.DIR_SOURCE_ROOT,
 380                                  'build/android/host_heartbeat.py')])
 381
 382
 383 def KillHostHeartbeat():
 384   ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
 385   stdout, _ = ps.communicate()
 386   matches = re.findall('\\n.*host_heartbeat.*', stdout)
 387   for match in matches:
 388     logging.info('An instance of host heart beart running... will kill')
 389     pid = re.findall(r'(\S+)', match)[1]
 390     subprocess.call(['kill', str(pid)])
 391
 392
 393 def main():
 394   # Recommended options on perf bots:
 395   # --disable-network
 396   #     TODO(tonyg): We eventually want network on. However, currently radios
 397   #     can cause perfbots to drain faster than they charge.
 398   # --min-battery-level 95
 399   #     Some perf bots run benchmarks with USB charging disabled which leads
 400   #     to gradual draining of the battery. We must wait for a full charge
 401   #     before starting a run in order to keep the devices online.
 402
 403   parser = argparse.ArgumentParser(
 404       description='Provision Android devices with settings required for bots.')
 405   parser.add_argument('-d', '--device', metavar='SERIAL',
 406                       help='the serial number of the device to be provisioned'
 407                       ' (the default is to provision all devices attached)')
 408   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 409   parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
 410                       dest='phases',
 411                       help='Phases of provisioning to run. '
 412                            '(If omitted, all phases will be run.)')
 413   parser.add_argument('--skip-wipe', action='store_true', default=False,
 414                       help="don't wipe device data during provisioning")
 415   parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
 416                       help='when wiping the device, max number of seconds to'
 417                       ' wait after each reboot '
 418                       '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
 419   parser.add_argument('--min-battery-level', type=int, metavar='NUM',
 420                       help='wait for the device to reach this minimum battery'
 421                       ' level before trying to continue')
 422   parser.add_argument('--disable-location', action='store_true',
 423                       help='disable Google location services on devices')
 424   parser.add_argument('--disable-mock-location', action='store_true',
 425                       default=False, help='Set ALLOW_MOCK_LOCATION to false')
 426   parser.add_argument('--disable-network', action='store_true',
 427                       help='disable network access on devices')
 428   parser.add_argument('--disable-java-debug', action='store_false',
 429                       dest='enable_java_debug', default=True,
 430                       help='disable Java property asserts and JNI checking')
 431   parser.add_argument('--disable-system-chrome', action='store_true',
 432                       help='Disable the system chrome from devices.')
 433   parser.add_argument('--remove-system-webview', action='store_true',
 434                       help='Remove the system webview from devices.')
 435   parser.add_argument('-t', '--target', default='Debug',
 436                       help='the build target (default: %(default)s)')
 437   parser.add_argument('-r', '--auto-reconnect', action='store_true',
 438                       help='push binary which will reboot the device on adb'
 439                       ' disconnections')
 440   parser.add_argument('--adb-key-files', type=str, nargs='+',
 441                       help='list of adb keys to push to device')
 442   parser.add_argument('-v', '--verbose', action='count', default=1,
 443                       help='Log more information.')
 444   parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
 445                       help='Wait for the battery to have this temp or lower.')
 446   parser.add_argument('--output-device-blacklist',
 447                       help='Json file to output the device blacklist.')
 448   parser.add_argument('--chrome-specific-wipe', action='store_true',
 449                       help='only wipe chrome specific data during provisioning')
 450   args = parser.parse_args()
 451   constants.SetBuildType(args.target)
 452
 453   run_tests_helper.SetLogLevel(args.verbose)
 454
 455   return ProvisionDevices(args)
 456
 457
 458 if __name__ == '__main__':
 459   sys.exit(main())