build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8
   9 import argparse
  10 import json
  11 import logging
  12 import os
  13 import psutil
  14 import re
  15 import signal
  16 import sys
  17
  18 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
  19 from devil.android import battery_utils
  20 from devil.android import device_blacklist
  21 from devil.android import device_errors
  22 from devil.android import device_list
  23 from devil.android import device_utils
  24 from devil.android.sdk import adb_wrapper
  25 from devil.utils import lsusb
  26 from devil.utils import reset_usb
  27 from devil.utils import run_tests_helper
  28 from pylib import constants
  29
  30 _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)')
  31
  32
  33 def KillAllAdb():
  34   def GetAllAdb():
  35     for p in psutil.process_iter():
  36       try:
  37         if 'adb' in p.name:
  38           yield p
  39       except (psutil.NoSuchProcess, psutil.AccessDenied):
  40         pass
  41
  42   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
  43     for p in GetAllAdb():
  44       try:
  45         logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
  46                      ' '.join(p.cmdline))
  47         p.send_signal(sig)
  48       except (psutil.NoSuchProcess, psutil.AccessDenied):
  49         pass
  50   for p in GetAllAdb():
  51     try:
  52       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
  53                     ' '.join(p.cmdline))
  54     except (psutil.NoSuchProcess, psutil.AccessDenied):
  55       pass
  56
  57
  58 def _BatteryStatus(device, blacklist):
  59   battery_info = {}
  60   try:
  61     battery = battery_utils.BatteryUtils(device)
  62     battery_info = battery.GetBatteryInfo(timeout=5)
  63     battery_level = int(battery_info.get('level', 100))
  64
  65     if battery_level < 15:
  66       logging.error('Critically low battery level (%d)', battery_level)
  67       battery = battery_utils.BatteryUtils(device)
  68       if not battery.GetCharging():
  69         battery.SetCharging(True)
  70       if blacklist:
  71         blacklist.Extend([device.GetDeviceSerial()])
  72
  73   except device_errors.CommandFailedError:
  74     logging.exception('Failed to get battery information for %s',
  75                       str(device))
  76
  77   return battery_info
  78
  79
  80 def _IMEISlice(device):
  81   imei_slice = ''
  82   try:
  83     for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  84                                     check_return=True, timeout=5):
  85       m = _RE_DEVICE_ID.match(l)
  86       if m:
  87         imei_slice = m.group(1)[-6:]
  88   except device_errors.CommandFailedError:
  89     logging.exception('Failed to get IMEI slice for %s', str(device))
  90
  91   return imei_slice
  92
  93
  94 def DeviceStatus(devices, blacklist):
  95   """Generates status information for the given devices.
  96
  97   Args:
  98     devices: The devices to generate status for.
  99     blacklist: The current device blacklist.
 100   Returns:
 101     A dict of the following form:
 102     {
 103       '<serial>': {
 104         'serial': '<serial>',
 105         'adb_status': str,
 106         'usb_status': bool,
 107         'blacklisted': bool,
 108         # only if the device is connected and not blacklisted
 109         'type': ro.build.product,
 110         'build': ro.build.id,
 111         'build_detail': ro.build.fingerprint,
 112         'battery': {
 113           ...
 114         },
 115         'imei_slice': str,
 116         'wifi_ip': str,
 117       },
 118       ...
 119     }
 120   """
 121   adb_devices = {
 122     a[0].GetDeviceSerial(): a
 123     for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True)
 124   }
 125   usb_devices = set(lsusb.get_android_devices())
 126
 127   def blacklisting_device_status(device):
 128     serial = device.adb.GetDeviceSerial()
 129     adb_status = (
 130         adb_devices[serial][1] if serial in adb_devices
 131         else 'unknown')
 132     usb_status = bool(serial in usb_devices)
 133
 134     device_status = {
 135       'serial': serial,
 136       'adb_status': adb_status,
 137       'usb_status': usb_status,
 138     }
 139
 140     if adb_status == 'device':
 141       if not serial in blacklist.Read():
 142         try:
 143           build_product = device.build_product
 144           build_id = device.build_id
 145           build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True)
 146           wifi_ip = device.GetProp('dhcp.wlan0.ipaddress')
 147           battery_info = _BatteryStatus(device, blacklist)
 148           imei_slice = _IMEISlice(device)
 149
 150           if (device.product_name == 'mantaray' and
 151               battery_info.get('AC powered', None) != 'true'):
 152             logging.error('Mantaray device not connected to AC power.')
 153
 154           device_status.update({
 155             'ro.build.product': build_product,
 156             'ro.build.id': build_id,
 157             'ro.build.fingerprint': build_fingerprint,
 158             'battery': battery_info,
 159             'imei_slice': imei_slice,
 160             'wifi_ip': wifi_ip,
 161
 162             # TODO(jbudorick): Remove these once no clients depend on them.
 163             'type': build_product,
 164             'build': build_id,
 165             'build_detail': build_fingerprint,
 166           })
 167
 168         except device_errors.CommandFailedError:
 169           logging.exception('Failure while getting device status for %s.',
 170                             str(device))
 171           if blacklist:
 172             blacklist.Extend([serial])
 173
 174         except device_errors.CommandTimeoutError:
 175           logging.exception('Timeout while getting device status for %s.',
 176                             str(device))
 177           if blacklist:
 178             blacklist.Extend([serial])
 179
 180     elif blacklist:
 181       blacklist.Extend([serial])
 182
 183     device_status['blacklisted'] = serial in blacklist.Read()
 184
 185     return device_status
 186
 187   parallel_devices = device_utils.DeviceUtils.parallel(devices)
 188   statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None)
 189   return statuses
 190
 191
 192 def RecoverDevices(devices, blacklist):
 193   """Attempts to recover any inoperable devices in the provided list.
 194
 195   Args:
 196     devices: The list of devices to attempt to recover.
 197     blacklist: The current device blacklist, which will be used then
 198       reset.
 199   Returns:
 200     Nothing.
 201   """
 202
 203   statuses = DeviceStatus(devices, blacklist)
 204
 205   should_restart_usb = set(
 206       status['serial'] for status in statuses
 207       if (not status['usb_status']
 208           or status['adb_status'] == 'unknown'))
 209   should_restart_adb = should_restart_usb.union(set(
 210       status['serial'] for status in statuses
 211       if status['adb_status'] in ('offline', 'unauthorized')))
 212   should_reboot_device = should_restart_adb.union(set(
 213       status['serial'] for status in statuses
 214       if status['blacklisted']))
 215
 216   logging.debug('Should restart USB for:')
 217   for d in should_restart_usb:
 218     logging.debug('  %s', d)
 219   logging.debug('Should restart ADB for:')
 220   for d in should_restart_adb:
 221     logging.debug('  %s', d)
 222   logging.debug('Should reboot:')
 223   for d in should_reboot_device:
 224     logging.debug('  %s', d)
 225
 226   blacklist.Reset()
 227
 228   if should_restart_adb:
 229     KillAllAdb()
 230   for serial in should_restart_usb:
 231     try:
 232       reset_usb.reset_android_usb(serial)
 233     except (IOError, device_errors.DeviceUnreachableError):
 234       logging.exception('Unable to reset USB for %s.', serial)
 235       if blacklist:
 236         blacklist.Extend([serial])
 237
 238   def blacklisting_recovery(device):
 239     if blacklist and device.adb.GetDeviceSerial() in blacklist.Read():
 240       logging.debug('%s is blacklisted, skipping recovery.', str(device))
 241       return
 242
 243     if device in should_reboot_device:
 244       try:
 245         device.WaitUntilFullyBooted()
 246         return
 247       except (device_errors.CommandTimeoutError,
 248               device_errors.CommandFailedError):
 249         logging.exception('Failure while waiting for %s. '
 250                           'Attempting to recover.', str(device))
 251
 252       try:
 253         device.Reboot()
 254         device.WaitUntilFullyBooted()
 255       except device_errors.CommandFailedError:
 256         logging.exception('Failure while waiting for %s.', str(device))
 257         if blacklist:
 258           blacklist.Extend([str(device)])
 259       except device_errors.CommandTimeoutError:
 260         logging.exception('Timed out while waiting for %s. ', str(device))
 261         if blacklist:
 262           blacklist.Extend([str(device)])
 263
 264   device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)
 265
 266
 267 def main():
 268   parser = argparse.ArgumentParser()
 269   parser.add_argument('--out-dir',
 270                       help='Directory where the device path is stored',
 271                       default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 272   parser.add_argument('--restart-usb', action='store_true',
 273                       help='DEPRECATED. '
 274                            'This script now always tries to reset USB.')
 275   parser.add_argument('--json-output',
 276                       help='Output JSON information into a specified file.')
 277   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 278   parser.add_argument('-v', '--verbose', action='count', default=1,
 279                       help='Log more information.')
 280
 281   args = parser.parse_args()
 282
 283   run_tests_helper.SetLogLevel(args.verbose)
 284
 285   if args.blacklist_file:
 286     blacklist = device_blacklist.Blacklist(args.blacklist_file)
 287   else:
 288     # TODO(jbudorick): Remove this once bots pass the blacklist file.
 289     blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON)
 290
 291   last_devices_path = os.path.join(
 292       args.out_dir, device_list.LAST_DEVICES_FILENAME)
 293   try:
 294     expected_devices = set(
 295         device_list.GetPersistentDeviceList(last_devices_path))
 296   except IOError:
 297     expected_devices = set()
 298
 299   usb_devices = set(lsusb.get_android_devices())
 300   devices = [device_utils.DeviceUtils(s)
 301              for s in expected_devices.union(usb_devices)]
 302
 303   RecoverDevices(devices, blacklist)
 304   statuses = DeviceStatus(devices, blacklist)
 305
 306   # Log the state of all devices.
 307   for status in statuses:
 308     logging.info(status['serial'])
 309     adb_status = status.get('adb_status')
 310     blacklisted = status.get('blacklisted')
 311     logging.info('  USB status: %s',
 312                  'online' if status.get('usb_status') else 'offline')
 313     logging.info('  ADB status: %s', adb_status)
 314     logging.info('  Blacklisted: %s', str(blacklisted))
 315     if adb_status == 'device' and not blacklisted:
 316       logging.info('  Device type: %s', status.get('ro.build.product'))
 317       logging.info('  OS build: %s', status.get('ro.build.id'))
 318       logging.info('  OS build fingerprint: %s',
 319                    status.get('ro.build.fingerprint'))
 320       logging.info('  Battery state:')
 321       for k, v in status.get('battery', {}).iteritems():
 322         logging.info('    %s: %s', k, v)
 323       logging.info('  IMEI slice: %s', status.get('imei_slice'))
 324       logging.info('  WiFi IP: %s', status.get('wifi_ip'))
 325
 326   # Update the last devices file.
 327   device_list.WritePersistentDeviceList(
 328       last_devices_path,
 329       [status['serial'] for status in statuses])
 330
 331   # Write device info to file for buildbot info display.
 332   if os.path.exists('/home/chrome-bot'):
 333     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 334       for status in statuses:
 335         try:
 336           if status['adb_status'] == 'device':
 337             f.write('{serial} {adb_status} {build_product} {build_id} '
 338                     '{temperature:.1f}C {level}%\n'.format(
 339                 serial=status['serial'],
 340                 adb_status=status['adb_status'],
 341                 build_product=status['type'],
 342                 build_id=status['build'],
 343                 temperature=float(status['battery']['temperature']) / 10,
 344                 level=status['battery']['level']
 345             ))
 346           else:
 347             f.write('{serial} {adb_status}'.format(
 348                 serial=status['serial'],
 349                 adb_status=status['adb_status']
 350             ))
 351         except Exception: # pylint: disable=broad-except
 352           pass
 353
 354   # Dump the device statuses to JSON.
 355   if args.json_output:
 356     with open(args.json_output, 'wb') as f:
 357       f.write(json.dumps(statuses, indent=4))
 358
 359   live_devices = [status['serial'] for status in statuses
 360                   if (status['adb_status'] == 'device'
 361                       and status['serial'] not in blacklist.Read())]
 362
 363   # If all devices failed, or if there are no devices, it's an infra error.
 364   return 0 if live_devices else constants.INFRA_EXIT_CODE
 365
 366
 367 if __name__ == '__main__':
 368   sys.exit(main())