build/android/provision_devices.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Provisions Android devices with settings required for bots.
   8
   9 Usage:
  10   ./provision_devices.py [-d <device serial number>]
  11 """
  12
  13 import argparse
  14 import datetime
  15 import json
  16 import logging
  17 import os
  18 import posixpath
  19 import re
  20 import subprocess
  21 import sys
  22 import time
  23
  24 from devil.android import battery_utils
  25 from devil.android import device_blacklist
  26 from devil.android import device_errors
  27 from devil.android import device_utils
  28 from devil.android.sdk import version_codes
  29 from devil.utils import run_tests_helper
  30 from devil.utils import timeout_retry
  31 from pylib import constants
  32 from pylib import device_settings
  33
  34 _SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
  35 _CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
  36 _TOMBSTONE_REGEX = re.compile('tombstone.*')
  37
  38
  39 class _DEFAULT_TIMEOUTS(object):
  40   # L can take a while to reboot after a wipe.
  41   LOLLIPOP = 600
  42   PRE_LOLLIPOP = 180
  43
  44   HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
  45
  46
  47 class _PHASES(object):
  48   WIPE = 'wipe'
  49   PROPERTIES = 'properties'
  50   FINISH = 'finish'
  51
  52   ALL = [WIPE, PROPERTIES, FINISH]
  53
  54
  55 def ProvisionDevices(args):
  56   if args.blacklist_file:
  57     blacklist = device_blacklist.Blacklist(args.blacklist_file)
  58   else:
  59     # TODO(jbudorick): Remove once the bots have switched over.
  60     blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON)
  61
  62   devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
  63   if args.device:
  64     devices = [d for d in devices if d == args.device]
  65     if not devices:
  66       raise device_errors.DeviceUnreachableError(args.device)
  67
  68   parallel_devices = device_utils.DeviceUtils.parallel(devices)
  69   parallel_devices.pMap(ProvisionDevice, blacklist, args)
  70   if args.auto_reconnect:
  71     _LaunchHostHeartbeat()
  72   blacklisted_devices = blacklist.Read()
  73   if args.output_device_blacklist:
  74     with open(args.output_device_blacklist, 'w') as f:
  75       json.dump(blacklisted_devices, f)
  76   if all(d in blacklisted_devices for d in devices):
  77     raise device_errors.NoDevicesError
  78   return 0
  79
  80
  81 def ProvisionDevice(device, blacklist, options):
  82   if options.reboot_timeout:
  83     reboot_timeout = options.reboot_timeout
  84   elif device.build_version_sdk >= version_codes.LOLLIPOP:
  85     reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
  86   else:
  87     reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
  88
  89   def should_run_phase(phase_name):
  90     return not options.phases or phase_name in options.phases
  91
  92   def run_phase(phase_func, reboot=True):
  93     try:
  94       device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
  95     except device_errors.CommandTimeoutError:
  96       logging.error('Device did not finish booting. Will try to reboot.')
  97       device.Reboot(timeout=reboot_timeout)
  98     phase_func(device, options)
  99     if reboot:
 100       device.Reboot(False, retries=0)
 101       device.adb.WaitForDevice()
 102
 103   try:
 104     if should_run_phase(_PHASES.WIPE):
 105       if options.chrome_specific_wipe:
 106         run_phase(WipeChromeData)
 107       else:
 108         run_phase(WipeDevice)
 109
 110     if should_run_phase(_PHASES.PROPERTIES):
 111       run_phase(SetProperties)
 112
 113     if should_run_phase(_PHASES.FINISH):
 114       run_phase(FinishProvisioning, reboot=False)
 115
 116   except device_errors.CommandTimeoutError:
 117     logging.exception('Timed out waiting for device %s. Adding to blacklist.',
 118                       str(device))
 119     blacklist.Extend([str(device)])
 120
 121   except device_errors.CommandFailedError:
 122     logging.exception('Failed to provision device %s. Adding to blacklist.',
 123                       str(device))
 124     blacklist.Extend([str(device)])
 125
 126
 127 def WipeChromeData(device, options):
 128   """Wipes chrome specific data from device
 129   Chrome specific data is:
 130   (1) any dir under /data/data/ whose name matches *chrom*, except
 131       com.android.chrome, which is the chrome stable package
 132   (2) any dir under /data/app/ and /data/app-lib/ whose name matches *chrom*
 133   (3) any files under /data/tombstones/ whose name matches "tombstone*"
 134   (4) /data/local.prop if there is any
 135   (5) /data/local/chrome-command-line if there is any
 136   (6) dir /data/local/.config/ if there is any (this is telemetry related)
 137   (7) dir /data/local/tmp/
 138
 139   Arguments:
 140     device: the device to wipe
 141   """
 142   if options.skip_wipe:
 143     return
 144
 145   try:
 146     device.EnableRoot()
 147     _WipeUnderDirIfMatch(device, '/data/data/', _CHROME_PACKAGE_REGEX,
 148                          constants.PACKAGE_INFO['chrome_stable'].package)
 149     _WipeUnderDirIfMatch(device, '/data/app/', _CHROME_PACKAGE_REGEX)
 150     _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
 151     _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
 152
 153     _WipeFileOrDir(device, '/data/local.prop')
 154     _WipeFileOrDir(device, '/data/local/chrome-command-line')
 155     _WipeFileOrDir(device, '/data/local/.config/')
 156     _WipeFileOrDir(device, '/data/local/tmp/')
 157
 158     device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
 159                            check_return=True)
 160   except device_errors.CommandFailedError:
 161     logging.exception('Possible failure while wiping the device. '
 162                       'Attempting to continue.')
 163
 164
 165 def WipeDevice(device, options):
 166   """Wipes data from device, keeping only the adb_keys for authorization.
 167
 168   After wiping data on a device that has been authorized, adb can still
 169   communicate with the device, but after reboot the device will need to be
 170   re-authorized because the adb keys file is stored in /data/misc/adb/.
 171   Thus, adb_keys file is rewritten so the device does not need to be
 172   re-authorized.
 173
 174   Arguments:
 175     device: the device to wipe
 176   """
 177   if options.skip_wipe:
 178     return
 179
 180   try:
 181     device.EnableRoot()
 182     device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
 183     if device_authorized:
 184       adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
 185                                  as_root=True).splitlines()
 186     device.RunShellCommand(['wipe', 'data'],
 187                            as_root=True, check_return=True)
 188     device.adb.WaitForDevice()
 189
 190     if device_authorized:
 191       adb_keys_set = set(adb_keys)
 192       for adb_key_file in options.adb_key_files or []:
 193         try:
 194           with open(adb_key_file, 'r') as f:
 195             adb_public_keys = f.readlines()
 196           adb_keys_set.update(adb_public_keys)
 197         except IOError:
 198           logging.warning('Unable to find adb keys file %s.', adb_key_file)
 199       _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
 200   except device_errors.CommandFailedError:
 201     logging.exception('Possible failure while wiping the device. '
 202                       'Attempting to continue.')
 203
 204
 205 def _WriteAdbKeysFile(device, adb_keys_string):
 206   dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
 207   device.RunShellCommand(['mkdir', '-p', dir_path],
 208                          as_root=True, check_return=True)
 209   device.RunShellCommand(['restorecon', dir_path],
 210                          as_root=True, check_return=True)
 211   device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
 212   device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
 213                          as_root=True, check_return=True)
 214
 215
 216 def SetProperties(device, options):
 217   try:
 218     device.EnableRoot()
 219   except device_errors.CommandFailedError as e:
 220     logging.warning(str(e))
 221
 222   _ConfigureLocalProperties(device, options.enable_java_debug)
 223   device_settings.ConfigureContentSettings(
 224       device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
 225   if options.disable_location:
 226     device_settings.ConfigureContentSettings(
 227         device, device_settings.DISABLE_LOCATION_SETTINGS)
 228   else:
 229     device_settings.ConfigureContentSettings(
 230         device, device_settings.ENABLE_LOCATION_SETTINGS)
 231
 232   if options.disable_mock_location:
 233     device_settings.ConfigureContentSettings(
 234         device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
 235   else:
 236     device_settings.ConfigureContentSettings(
 237         device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
 238
 239   device_settings.SetLockScreenSettings(device)
 240   if options.disable_network:
 241     device_settings.ConfigureContentSettings(
 242         device, device_settings.NETWORK_DISABLED_SETTINGS)
 243
 244   if options.disable_system_chrome:
 245     # The system chrome version on the device interferes with some tests.
 246     device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
 247                            check_return=True)
 248
 249   if options.remove_system_webview:
 250     if device.HasRoot():
 251       # This is required, e.g., to replace the system webview on a device.
 252       device.adb.Remount()
 253       device.RunShellCommand(['stop'], check_return=True)
 254       device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
 255                              check_return=True)
 256       device.RunShellCommand(['start'], check_return=True)
 257     else:
 258       logging.warning('Cannot remove system webview from a non-rooted device')
 259
 260
 261 def _ConfigureLocalProperties(device, java_debug=True):
 262   """Set standard readonly testing device properties prior to reboot."""
 263   local_props = [
 264       'persist.sys.usb.config=adb',
 265       'ro.monkey=1',
 266       'ro.test_harness=1',
 267       'ro.audio.silent=1',
 268       'ro.setupwizard.mode=DISABLED',
 269       ]
 270   if java_debug:
 271     local_props.append(
 272         '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
 273     local_props.append('debug.checkjni=1')
 274   try:
 275     device.WriteFile(
 276         device.LOCAL_PROPERTIES_PATH,
 277         '\n'.join(local_props), as_root=True)
 278     # Android will not respect the local props file if it is world writable.
 279     device.RunShellCommand(
 280         ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
 281         as_root=True, check_return=True)
 282   except device_errors.CommandFailedError:
 283     logging.exception('Failed to configure local properties.')
 284
 285
 286 def FinishProvisioning(device, options):
 287   if options.min_battery_level is not None:
 288     try:
 289       battery = battery_utils.BatteryUtils(device)
 290       battery.ChargeDeviceToLevel(options.min_battery_level)
 291     except device_errors.CommandFailedError:
 292       logging.exception('Unable to charge device to specified level.')
 293
 294   if options.max_battery_temp is not None:
 295     try:
 296       battery = battery_utils.BatteryUtils(device)
 297       battery.LetBatteryCoolToTemperature(options.max_battery_temp)
 298     except device_errors.CommandFailedError:
 299       logging.exception('Unable to let battery cool to specified temperature.')
 300
 301   def _set_and_verify_date():
 302     if device.build_version_sdk >= version_codes.MARSHMALLOW:
 303       date_format = '%m%d%H%M%Y.%S'
 304       set_date_command = ['date']
 305     else:
 306       date_format = '%Y%m%d.%H%M%S'
 307       set_date_command = ['date', '-s']
 308     strgmtime = time.strftime(date_format, time.gmtime())
 309     set_date_command.append(strgmtime)
 310     device.RunShellCommand(set_date_command, as_root=True, check_return=True)
 311
 312     device_time = device.RunShellCommand(
 313         ['date', '+"%Y%m%d.%H%M%S"'], as_root=True,
 314         single_line=True).replace('"', '')
 315     device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
 316     correct_time = datetime.datetime.strptime(strgmtime, date_format)
 317     tdelta = (correct_time - device_time).seconds
 318     if tdelta <= 1:
 319       logging.info('Date/time successfully set on %s', device)
 320       return True
 321     else:
 322       logging.error('Date mismatch. Device: %s Correct: %s',
 323                     device_time.isoformat(), correct_time.isoformat())
 324       return False
 325
 326   # Sometimes the date is not set correctly on the devices. Retry on failure.
 327   if not timeout_retry.WaitFor(_set_and_verify_date, wait_period=1,
 328                                max_tries=2):
 329     raise device_errors.CommandFailedError(
 330         'Failed to set date & time.', device_serial=str(device))
 331
 332   props = device.RunShellCommand('getprop', check_return=True)
 333   for prop in props:
 334     logging.info('  %s', prop)
 335   if options.auto_reconnect:
 336     _PushAndLaunchAdbReboot(device, options.target)
 337
 338
 339 def _WipeUnderDirIfMatch(device, path, pattern, app_to_keep=None):
 340   ls_result = device.Ls(path)
 341   for (content, _) in ls_result:
 342     if pattern.match(content):
 343       if content != app_to_keep:
 344         _WipeFileOrDir(device, path + content)
 345
 346
 347 def _WipeFileOrDir(device, path):
 348   if device.PathExists(path):
 349     device.RunShellCommand(['rm', '-rf', path], check_return=True)
 350
 351
 352 def _PushAndLaunchAdbReboot(device, target):
 353   """Pushes and launches the adb_reboot binary on the device.
 354
 355   Arguments:
 356     device: The DeviceUtils instance for the device to which the adb_reboot
 357             binary should be pushed.
 358     target: The build target (example, Debug or Release) which helps in
 359             locating the adb_reboot binary.
 360   """
 361   logging.info('Will push and launch adb_reboot on %s', str(device))
 362   # Kill if adb_reboot is already running.
 363   device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
 364   # Push adb_reboot
 365   logging.info('  Pushing adb_reboot ...')
 366   adb_reboot = os.path.join(constants.DIR_SOURCE_ROOT,
 367                             'out/%s/adb_reboot' % target)
 368   device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
 369   # Launch adb_reboot
 370   logging.info('  Launching adb_reboot ...')
 371   device.RunShellCommand(
 372       ['/data/local/tmp/adb_reboot'],
 373       check_return=True)
 374
 375
 376 def _LaunchHostHeartbeat():
 377   # Kill if existing host_heartbeat
 378   KillHostHeartbeat()
 379   # Launch a new host_heartbeat
 380   logging.info('Spawning host heartbeat...')
 381   subprocess.Popen([os.path.join(constants.DIR_SOURCE_ROOT,
 382                                  'build/android/host_heartbeat.py')])
 383
 384
 385 def KillHostHeartbeat():
 386   ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
 387   stdout, _ = ps.communicate()
 388   matches = re.findall('\\n.*host_heartbeat.*', stdout)
 389   for match in matches:
 390     logging.info('An instance of host heart beart running... will kill')
 391     pid = re.findall(r'(\S+)', match)[1]
 392     subprocess.call(['kill', str(pid)])
 393
 394
 395 def main():
 396   # Recommended options on perf bots:
 397   # --disable-network
 398   #     TODO(tonyg): We eventually want network on. However, currently radios
 399   #     can cause perfbots to drain faster than they charge.
 400   # --min-battery-level 95
 401   #     Some perf bots run benchmarks with USB charging disabled which leads
 402   #     to gradual draining of the battery. We must wait for a full charge
 403   #     before starting a run in order to keep the devices online.
 404
 405   parser = argparse.ArgumentParser(
 406       description='Provision Android devices with settings required for bots.')
 407   parser.add_argument('-d', '--device', metavar='SERIAL',
 408                       help='the serial number of the device to be provisioned'
 409                       ' (the default is to provision all devices attached)')
 410   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 411   parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
 412                       dest='phases',
 413                       help='Phases of provisioning to run. '
 414                            '(If omitted, all phases will be run.)')
 415   parser.add_argument('--skip-wipe', action='store_true', default=False,
 416                       help="don't wipe device data during provisioning")
 417   parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
 418                       help='when wiping the device, max number of seconds to'
 419                       ' wait after each reboot '
 420                       '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
 421   parser.add_argument('--min-battery-level', type=int, metavar='NUM',
 422                       help='wait for the device to reach this minimum battery'
 423                       ' level before trying to continue')
 424   parser.add_argument('--disable-location', action='store_true',
 425                       help='disable Google location services on devices')
 426   parser.add_argument('--disable-mock-location', action='store_true',
 427                       default=False, help='Set ALLOW_MOCK_LOCATION to false')
 428   parser.add_argument('--disable-network', action='store_true',
 429                       help='disable network access on devices')
 430   parser.add_argument('--disable-java-debug', action='store_false',
 431                       dest='enable_java_debug', default=True,
 432                       help='disable Java property asserts and JNI checking')
 433   parser.add_argument('--disable-system-chrome', action='store_true',
 434                       help='Disable the system chrome from devices.')
 435   parser.add_argument('--remove-system-webview', action='store_true',
 436                       help='Remove the system webview from devices.')
 437   parser.add_argument('-t', '--target', default='Debug',
 438                       help='the build target (default: %(default)s)')
 439   parser.add_argument('-r', '--auto-reconnect', action='store_true',
 440                       help='push binary which will reboot the device on adb'
 441                       ' disconnections')
 442   parser.add_argument('--adb-key-files', type=str, nargs='+',
 443                       help='list of adb keys to push to device')
 444   parser.add_argument('-v', '--verbose', action='count', default=1,
 445                       help='Log more information.')
 446   parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
 447                       help='Wait for the battery to have this temp or lower.')
 448   parser.add_argument('--output-device-blacklist',
 449                       help='Json file to output the device blacklist.')
 450   parser.add_argument('--chrome-specific-wipe', action='store_true',
 451                       help='only wipe chrome specific data during provisioning')
 452   args = parser.parse_args()
 453   constants.SetBuildType(args.target)
 454
 455   run_tests_helper.SetLogLevel(args.verbose)
 456
 457   return ProvisionDevices(args)
 458
 459
 460 if __name__ == '__main__':
 461   sys.exit(main())