build/android/provision_devices.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Provisions Android devices with settings required for bots.
   8
   9 Usage:
  10   ./provision_devices.py [-d <device serial number>]
  11 """
  12
  13 import argparse
  14 import datetime
  15 import json
  16 import logging
  17 import os
  18 import posixpath
  19 import re
  20 import subprocess
  21 import sys
  22 import time
  23
  24 from devil.android import battery_utils
  25 from devil.android import device_blacklist
  26 from devil.android import device_errors
  27 from devil.android import device_utils
  28 from devil.android.sdk import version_codes
  29 from devil.utils import run_tests_helper
  30 from devil.utils import timeout_retry
  31 from pylib import constants
  32 from pylib import device_settings
  33
  34 _SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
  35 _CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
  36 _TOMBSTONE_REGEX = re.compile('tombstone.*')
  37
  38
  39 class _DEFAULT_TIMEOUTS(object):
  40   # L can take a while to reboot after a wipe.
  41   LOLLIPOP = 600
  42   PRE_LOLLIPOP = 180
  43
  44   HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
  45
  46
  47 class _PHASES(object):
  48   WIPE = 'wipe'
  49   PROPERTIES = 'properties'
  50   FINISH = 'finish'
  51
  52   ALL = [WIPE, PROPERTIES, FINISH]
  53
  54
  55 def ProvisionDevices(args):
  56   if args.blacklist_file:
  57     blacklist = device_blacklist.Blacklist(args.blacklist_file)
  58   else:
  59     # TODO(jbudorick): Remove once the bots have switched over.
  60     blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON)
  61
  62   devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
  63   if args.device:
  64     devices = [d for d in devices if d == args.device]
  65     if not devices:
  66       raise device_errors.DeviceUnreachableError(args.device)
  67
  68   parallel_devices = device_utils.DeviceUtils.parallel(devices)
  69   parallel_devices.pMap(ProvisionDevice, blacklist, args)
  70   if args.auto_reconnect:
  71     _LaunchHostHeartbeat()
  72   blacklisted_devices = blacklist.Read()
  73   if args.output_device_blacklist:
  74     with open(args.output_device_blacklist, 'w') as f:
  75       json.dump(blacklisted_devices, f)
  76   if all(d in blacklisted_devices for d in devices):
  77     raise device_errors.NoDevicesError
  78   return 0
  79
  80
  81 def ProvisionDevice(device, blacklist, options):
  82   if options.reboot_timeout:
  83     reboot_timeout = options.reboot_timeout
  84   elif device.build_version_sdk >= version_codes.LOLLIPOP:
  85     reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
  86   else:
  87     reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
  88
  89   def should_run_phase(phase_name):
  90     return not options.phases or phase_name in options.phases
  91
  92   def run_phase(phase_func, reboot=True):
  93     try:
  94       device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
  95     except device_errors.CommandTimeoutError:
  96       logging.error('Device did not finish booting. Will try to reboot.')
  97       device.Reboot(timeout=reboot_timeout)
  98     phase_func(device, options)
  99     if reboot:
 100       device.Reboot(False, retries=0)
 101       device.adb.WaitForDevice()
 102
 103   try:
 104     if should_run_phase(_PHASES.WIPE):
 105       if options.chrome_specific_wipe:
 106         run_phase(WipeChromeData)
 107       else:
 108         run_phase(WipeDevice)
 109
 110     if should_run_phase(_PHASES.PROPERTIES):
 111       run_phase(SetProperties)
 112
 113     if should_run_phase(_PHASES.FINISH):
 114       run_phase(FinishProvisioning, reboot=False)
 115
 116   except device_errors.CommandTimeoutError:
 117     logging.exception('Timed out waiting for device %s. Adding to blacklist.',
 118                       str(device))
 119     blacklist.Extend([str(device)])
 120
 121   except device_errors.CommandFailedError:
 122     logging.exception('Failed to provision device %s. Adding to blacklist.',
 123                       str(device))
 124     blacklist.Extend([str(device)])
 125
 126
 127 def WipeChromeData(device, options):
 128   """Wipes chrome specific data from device
 129   Chrome specific data is:
 130   (1) any dir under /data/data/ whose name matches *chrom*, except
 131       com.android.chrome, which is the chrome stable package
 132   (2) any dir under /data/app/ and /data/app-lib/ whose name matches *chrom*
 133   (3) any files under /data/tombstones/ whose name matches "tombstone*"
 134   (4) /data/local.prop if there is any
 135   (5) /data/local/chrome-command-line if there is any
 136   (6) dir /data/local/.config/ if there is any (this is telemetry related)
 137   (7) dir /data/local/tmp/
 138
 139   Arguments:
 140     device: the device to wipe
 141   """
 142   if options.skip_wipe:
 143     return
 144
 145   try:
 146     device.EnableRoot()
 147     _WipeUnderDirIfMatch(device, '/data/data/', _CHROME_PACKAGE_REGEX,
 148                          constants.PACKAGE_INFO['chrome_stable'].package)
 149     _WipeUnderDirIfMatch(device, '/data/app/', _CHROME_PACKAGE_REGEX)
 150     _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
 151     _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
 152
 153     _WipeFileOrDir(device, '/data/local.prop')
 154     _WipeFileOrDir(device, '/data/local/chrome-command-line')
 155     _WipeFileOrDir(device, '/data/local/.config/')
 156     _WipeFileOrDir(device, '/data/local/tmp/')
 157
 158     device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
 159                            check_return=True)
 160   except device_errors.CommandFailedError:
 161     logging.exception('Possible failure while wiping the device. '
 162                       'Attempting to continue.')
 163
 164
 165 def WipeDevice(device, options):
 166   """Wipes data from device, keeping only the adb_keys for authorization.
 167
 168   After wiping data on a device that has been authorized, adb can still
 169   communicate with the device, but after reboot the device will need to be
 170   re-authorized because the adb keys file is stored in /data/misc/adb/.
 171   Thus, adb_keys file is rewritten so the device does not need to be
 172   re-authorized.
 173
 174   Arguments:
 175     device: the device to wipe
 176   """
 177   if options.skip_wipe:
 178     return
 179
 180   try:
 181     device.EnableRoot()
 182     device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
 183     if device_authorized:
 184       adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
 185                                  as_root=True).splitlines()
 186     device.RunShellCommand(['wipe', 'data'],
 187                            as_root=True, check_return=True)
 188     device.adb.WaitForDevice()
 189
 190     if device_authorized:
 191       adb_keys_set = set(adb_keys)
 192       for adb_key_file in options.adb_key_files or []:
 193         try:
 194           with open(adb_key_file, 'r') as f:
 195             adb_public_keys = f.readlines()
 196           adb_keys_set.update(adb_public_keys)
 197         except IOError:
 198           logging.warning('Unable to find adb keys file %s.', adb_key_file)
 199       _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
 200   except device_errors.CommandFailedError:
 201     logging.exception('Possible failure while wiping the device. '
 202                       'Attempting to continue.')
 203
 204
 205 def _WriteAdbKeysFile(device, adb_keys_string):
 206   dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
 207   device.RunShellCommand(['mkdir', '-p', dir_path],
 208                          as_root=True, check_return=True)
 209   device.RunShellCommand(['restorecon', dir_path],
 210                          as_root=True, check_return=True)
 211   device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
 212   device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
 213                          as_root=True, check_return=True)
 214
 215
 216 def SetProperties(device, options):
 217   try:
 218     device.EnableRoot()
 219   except device_errors.CommandFailedError as e:
 220     logging.warning(str(e))
 221
 222   _ConfigureLocalProperties(device, options.enable_java_debug)
 223   device_settings.ConfigureContentSettings(
 224       device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
 225   if options.disable_location:
 226     device_settings.ConfigureContentSettings(
 227         device, device_settings.DISABLE_LOCATION_SETTINGS)
 228   else:
 229     device_settings.ConfigureContentSettings(
 230         device, device_settings.ENABLE_LOCATION_SETTINGS)
 231
 232   if options.disable_mock_location:
 233     device_settings.ConfigureContentSettings(
 234         device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
 235   else:
 236     device_settings.ConfigureContentSettings(
 237         device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
 238
 239   device_settings.SetLockScreenSettings(device)
 240   if options.disable_network:
 241     device_settings.ConfigureContentSettings(
 242         device, device_settings.NETWORK_DISABLED_SETTINGS)
 243
 244   if options.disable_system_chrome:
 245     # The system chrome version on the device interferes with some tests.
 246     device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
 247                            check_return=True)
 248
 249   if options.remove_system_webview:
 250     if device.HasRoot():
 251       # This is required, e.g., to replace the system webview on a device.
 252       device.adb.Remount()
 253       device.RunShellCommand(['stop'], check_return=True)
 254       device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
 255                              check_return=True)
 256       device.RunShellCommand(['start'], check_return=True)
 257     else:
 258       logging.warning('Cannot remove system webview from a non-rooted device')
 259
 260
 261 def _ConfigureLocalProperties(device, java_debug=True):
 262   """Set standard readonly testing device properties prior to reboot."""
 263   local_props = [
 264       'persist.sys.usb.config=adb',
 265       'ro.monkey=1',
 266       'ro.test_harness=1',
 267       'ro.audio.silent=1',
 268       'ro.setupwizard.mode=DISABLED',
 269       ]
 270   if java_debug:
 271     local_props.append(
 272         '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
 273     local_props.append('debug.checkjni=1')
 274   try:
 275     device.WriteFile(
 276         device.LOCAL_PROPERTIES_PATH,
 277         '\n'.join(local_props), as_root=True)
 278     # Android will not respect the local props file if it is world writable.
 279     device.RunShellCommand(
 280         ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
 281         as_root=True, check_return=True)
 282   except device_errors.CommandFailedError:
 283     logging.exception('Failed to configure local properties.')
 284
 285
 286 def FinishProvisioning(device, options):
 287   if options.min_battery_level is not None:
 288     try:
 289       battery = battery_utils.BatteryUtils(device)
 290       battery.ChargeDeviceToLevel(options.min_battery_level)
 291     except device_errors.CommandFailedError:
 292       logging.exception('Unable to charge device to specified level.')
 293
 294   if options.max_battery_temp is not None:
 295     try:
 296       battery = battery_utils.BatteryUtils(device)
 297       battery.LetBatteryCoolToTemperature(options.max_battery_temp)
 298     except device_errors.CommandFailedError:
 299       logging.exception('Unable to let battery cool to specified temperature.')
 300
 301   def _set_and_verify_date():
 302     if device.build_version_sdk >= version_codes.MARSHMALLOW:
 303       date_format = '%m%d%H%M%Y.%S'
 304       set_date_command = ['date']
 305     else:
 306       date_format = '%Y%m%d.%H%M%S'
 307       set_date_command = ['date', '-s']
 308     strgmtime = time.strftime(date_format, time.gmtime())
 309     set_date_command.append(strgmtime)
 310     device.RunShellCommand(set_date_command, as_root=True, check_return=True)
 311
 312     device_time = device.RunShellCommand(
 313         ['date', '+"%Y%m%d.%H%M%S"'], as_root=True,
 314         single_line=True).replace('"', '')
 315     device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
 316     correct_time = datetime.datetime.strptime(strgmtime, date_format)
 317     tdelta = (correct_time - device_time).seconds
 318     if tdelta <= 1:
 319       logging.info('Date/time successfully set on %s', device)
 320       return True
 321     else:
 322       return False
 323
 324   # Sometimes the date is not set correctly on the devices. Retry on failure.
 325   if not timeout_retry.WaitFor(_set_and_verify_date, wait_period=1,
 326                                max_tries=2):
 327     raise device_errors.CommandFailedError(
 328         'Failed to set date & time.', device_serial=str(device))
 329
 330   props = device.RunShellCommand('getprop', check_return=True)
 331   for prop in props:
 332     logging.info('  %s', prop)
 333   if options.auto_reconnect:
 334     _PushAndLaunchAdbReboot(device, options.target)
 335
 336
 337 def _WipeUnderDirIfMatch(device, path, pattern, app_to_keep=None):
 338   ls_result = device.Ls(path)
 339   for (content, _) in ls_result:
 340     if pattern.match(content):
 341       if content != app_to_keep:
 342         _WipeFileOrDir(device, path + content)
 343
 344
 345 def _WipeFileOrDir(device, path):
 346   if device.PathExists(path):
 347     device.RunShellCommand(['rm', '-rf', path], check_return=True)
 348
 349
 350 def _PushAndLaunchAdbReboot(device, target):
 351   """Pushes and launches the adb_reboot binary on the device.
 352
 353   Arguments:
 354     device: The DeviceUtils instance for the device to which the adb_reboot
 355             binary should be pushed.
 356     target: The build target (example, Debug or Release) which helps in
 357             locating the adb_reboot binary.
 358   """
 359   logging.info('Will push and launch adb_reboot on %s', str(device))
 360   # Kill if adb_reboot is already running.
 361   device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
 362   # Push adb_reboot
 363   logging.info('  Pushing adb_reboot ...')
 364   adb_reboot = os.path.join(constants.DIR_SOURCE_ROOT,
 365                             'out/%s/adb_reboot' % target)
 366   device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
 367   # Launch adb_reboot
 368   logging.info('  Launching adb_reboot ...')
 369   device.RunShellCommand(
 370       ['/data/local/tmp/adb_reboot'],
 371       check_return=True)
 372
 373
 374 def _LaunchHostHeartbeat():
 375   # Kill if existing host_heartbeat
 376   KillHostHeartbeat()
 377   # Launch a new host_heartbeat
 378   logging.info('Spawning host heartbeat...')
 379   subprocess.Popen([os.path.join(constants.DIR_SOURCE_ROOT,
 380                                  'build/android/host_heartbeat.py')])
 381
 382
 383 def KillHostHeartbeat():
 384   ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
 385   stdout, _ = ps.communicate()
 386   matches = re.findall('\\n.*host_heartbeat.*', stdout)
 387   for match in matches:
 388     logging.info('An instance of host heart beart running... will kill')
 389     pid = re.findall(r'(\S+)', match)[1]
 390     subprocess.call(['kill', str(pid)])
 391
 392
 393 def main():
 394   # Recommended options on perf bots:
 395   # --disable-network
 396   #     TODO(tonyg): We eventually want network on. However, currently radios
 397   #     can cause perfbots to drain faster than they charge.
 398   # --min-battery-level 95
 399   #     Some perf bots run benchmarks with USB charging disabled which leads
 400   #     to gradual draining of the battery. We must wait for a full charge
 401   #     before starting a run in order to keep the devices online.
 402
 403   parser = argparse.ArgumentParser(
 404       description='Provision Android devices with settings required for bots.')
 405   parser.add_argument('-d', '--device', metavar='SERIAL',
 406                       help='the serial number of the device to be provisioned'
 407                       ' (the default is to provision all devices attached)')
 408   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 409   parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
 410                       dest='phases',
 411                       help='Phases of provisioning to run. '
 412                            '(If omitted, all phases will be run.)')
 413   parser.add_argument('--skip-wipe', action='store_true', default=False,
 414                       help="don't wipe device data during provisioning")
 415   parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
 416                       help='when wiping the device, max number of seconds to'
 417                       ' wait after each reboot '
 418                       '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
 419   parser.add_argument('--min-battery-level', type=int, metavar='NUM',
 420                       help='wait for the device to reach this minimum battery'
 421                       ' level before trying to continue')
 422   parser.add_argument('--disable-location', action='store_true',
 423                       help='disable Google location services on devices')
 424   parser.add_argument('--disable-mock-location', action='store_true',
 425                       default=False, help='Set ALLOW_MOCK_LOCATION to false')
 426   parser.add_argument('--disable-network', action='store_true',
 427                       help='disable network access on devices')
 428   parser.add_argument('--disable-java-debug', action='store_false',
 429                       dest='enable_java_debug', default=True,
 430                       help='disable Java property asserts and JNI checking')
 431   parser.add_argument('--disable-system-chrome', action='store_true',
 432                       help='Disable the system chrome from devices.')
 433   parser.add_argument('--remove-system-webview', action='store_true',
 434                       help='Remove the system webview from devices.')
 435   parser.add_argument('-t', '--target', default='Debug',
 436                       help='the build target (default: %(default)s)')
 437   parser.add_argument('-r', '--auto-reconnect', action='store_true',
 438                       help='push binary which will reboot the device on adb'
 439                       ' disconnections')
 440   parser.add_argument('--adb-key-files', type=str, nargs='+',
 441                       help='list of adb keys to push to device')
 442   parser.add_argument('-v', '--verbose', action='count', default=1,
 443                       help='Log more information.')
 444   parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
 445                       help='Wait for the battery to have this temp or lower.')
 446   parser.add_argument('--output-device-blacklist',
 447                       help='Json file to output the device blacklist.')
 448   parser.add_argument('--chrome-specific-wipe', action='store_true',
 449                       help='only wipe chrome specific data during provisioning')
 450   args = parser.parse_args()
 451   constants.SetBuildType(args.target)
 452
 453   run_tests_helper.SetLogLevel(args.verbose)
 454
 455   return ProvisionDevices(args)
 456
 457
 458 if __name__ == '__main__':
 459   sys.exit(main())