build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8
   9 import argparse
  10 import json
  11 import logging
  12 import os
  13 import psutil
  14 import re
  15 import signal
  16 import sys
  17
  18 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
  19 from devil.android import battery_utils
  20 from devil.android import device_blacklist
  21 from devil.android import device_errors
  22 from devil.android import device_list
  23 from devil.android import device_utils
  24 from devil.android.sdk import adb_wrapper
  25 from devil.utils import lsusb
  26 from devil.utils import reset_usb
  27 from devil.utils import run_tests_helper
  28 from pylib import constants
  29
  30 _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)')
  31
  32
  33 def KillAllAdb():
  34   def GetAllAdb():
  35     for p in psutil.process_iter():
  36       try:
  37         if 'adb' in p.name:
  38           yield p
  39       except (psutil.NoSuchProcess, psutil.AccessDenied):
  40         pass
  41
  42   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
  43     for p in GetAllAdb():
  44       try:
  45         logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
  46                      ' '.join(p.cmdline))
  47         p.send_signal(sig)
  48       except (psutil.NoSuchProcess, psutil.AccessDenied):
  49         pass
  50   for p in GetAllAdb():
  51     try:
  52       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
  53                     ' '.join(p.cmdline))
  54     except (psutil.NoSuchProcess, psutil.AccessDenied):
  55       pass
  56
  57
  58 def _IsBlacklisted(serial, blacklist):
  59   return blacklist and serial in blacklist.Read()
  60
  61
  62 def _BatteryStatus(device, blacklist):
  63   battery_info = {}
  64   try:
  65     battery = battery_utils.BatteryUtils(device)
  66     battery_info = battery.GetBatteryInfo(timeout=5)
  67     battery_level = int(battery_info.get('level', 100))
  68
  69     if battery_level < 15:
  70       logging.error('Critically low battery level (%d)', battery_level)
  71       battery = battery_utils.BatteryUtils(device)
  72       if not battery.GetCharging():
  73         battery.SetCharging(True)
  74       if blacklist:
  75         blacklist.Extend([device.adb.GetDeviceSerial()])
  76
  77   except device_errors.CommandFailedError:
  78     logging.exception('Failed to get battery information for %s',
  79                       str(device))
  80
  81   return battery_info
  82
  83
  84 def _IMEISlice(device):
  85   imei_slice = ''
  86   try:
  87     for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  88                                     check_return=True, timeout=5):
  89       m = _RE_DEVICE_ID.match(l)
  90       if m:
  91         imei_slice = m.group(1)[-6:]
  92   except device_errors.CommandFailedError:
  93     logging.exception('Failed to get IMEI slice for %s', str(device))
  94
  95   return imei_slice
  96
  97
  98 def DeviceStatus(devices, blacklist):
  99   """Generates status information for the given devices.
 100
 101   Args:
 102     devices: The devices to generate status for.
 103     blacklist: The current device blacklist.
 104   Returns:
 105     A dict of the following form:
 106     {
 107       '<serial>': {
 108         'serial': '<serial>',
 109         'adb_status': str,
 110         'usb_status': bool,
 111         'blacklisted': bool,
 112         # only if the device is connected and not blacklisted
 113         'type': ro.build.product,
 114         'build': ro.build.id,
 115         'build_detail': ro.build.fingerprint,
 116         'battery': {
 117           ...
 118         },
 119         'imei_slice': str,
 120         'wifi_ip': str,
 121       },
 122       ...
 123     }
 124   """
 125   adb_devices = {
 126     a[0].GetDeviceSerial(): a
 127     for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True)
 128   }
 129   usb_devices = set(lsusb.get_android_devices())
 130
 131   def blacklisting_device_status(device):
 132     serial = device.adb.GetDeviceSerial()
 133     adb_status = (
 134         adb_devices[serial][1] if serial in adb_devices
 135         else 'unknown')
 136     usb_status = bool(serial in usb_devices)
 137
 138     device_status = {
 139       'serial': serial,
 140       'adb_status': adb_status,
 141       'usb_status': usb_status,
 142     }
 143
 144     if adb_status == 'device':
 145       if not _IsBlacklisted(serial, blacklist):
 146         try:
 147           build_product = device.build_product
 148           build_id = device.build_id
 149           build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True)
 150           wifi_ip = device.GetProp('dhcp.wlan0.ipaddress')
 151           battery_info = _BatteryStatus(device, blacklist)
 152           imei_slice = _IMEISlice(device)
 153
 154           if (device.product_name == 'mantaray' and
 155               battery_info.get('AC powered', None) != 'true'):
 156             logging.error('Mantaray device not connected to AC power.')
 157
 158           device_status.update({
 159             'ro.build.product': build_product,
 160             'ro.build.id': build_id,
 161             'ro.build.fingerprint': build_fingerprint,
 162             'battery': battery_info,
 163             'imei_slice': imei_slice,
 164             'wifi_ip': wifi_ip,
 165
 166             # TODO(jbudorick): Remove these once no clients depend on them.
 167             'type': build_product,
 168             'build': build_id,
 169             'build_detail': build_fingerprint,
 170           })
 171
 172         except device_errors.CommandFailedError:
 173           logging.exception('Failure while getting device status for %s.',
 174                             str(device))
 175           if blacklist:
 176             blacklist.Extend([serial])
 177
 178         except device_errors.CommandTimeoutError:
 179           logging.exception('Timeout while getting device status for %s.',
 180                             str(device))
 181           if blacklist:
 182             blacklist.Extend([serial])
 183
 184     elif blacklist:
 185       blacklist.Extend([serial])
 186
 187     device_status['blacklisted'] = _IsBlacklisted(serial, blacklist)
 188
 189     return device_status
 190
 191   parallel_devices = device_utils.DeviceUtils.parallel(devices)
 192   statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None)
 193   return statuses
 194
 195
 196 def RecoverDevices(devices, blacklist):
 197   """Attempts to recover any inoperable devices in the provided list.
 198
 199   Args:
 200     devices: The list of devices to attempt to recover.
 201     blacklist: The current device blacklist, which will be used then
 202       reset.
 203   Returns:
 204     Nothing.
 205   """
 206
 207   statuses = DeviceStatus(devices, blacklist)
 208
 209   should_restart_usb = set(
 210       status['serial'] for status in statuses
 211       if (not status['usb_status']
 212           or status['adb_status'] == 'unknown'))
 213   should_restart_adb = should_restart_usb.union(set(
 214       status['serial'] for status in statuses
 215       if status['adb_status'] in ('offline', 'unauthorized')))
 216   should_reboot_device = should_restart_adb.union(set(
 217       status['serial'] for status in statuses
 218       if status['blacklisted']))
 219
 220   logging.debug('Should restart USB for:')
 221   for d in should_restart_usb:
 222     logging.debug('  %s', d)
 223   logging.debug('Should restart ADB for:')
 224   for d in should_restart_adb:
 225     logging.debug('  %s', d)
 226   logging.debug('Should reboot:')
 227   for d in should_reboot_device:
 228     logging.debug('  %s', d)
 229
 230   if blacklist:
 231     blacklist.Reset()
 232
 233   if should_restart_adb:
 234     KillAllAdb()
 235   for serial in should_restart_usb:
 236     try:
 237       reset_usb.reset_android_usb(serial)
 238     except (IOError, device_errors.DeviceUnreachableError):
 239       logging.exception('Unable to reset USB for %s.', serial)
 240       if blacklist:
 241         blacklist.Extend([serial])
 242
 243   def blacklisting_recovery(device):
 244     if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
 245       logging.debug('%s is blacklisted, skipping recovery.', str(device))
 246       return
 247
 248     if device in should_reboot_device:
 249       try:
 250         device.WaitUntilFullyBooted()
 251         return
 252       except (device_errors.CommandTimeoutError,
 253               device_errors.CommandFailedError):
 254         logging.exception('Failure while waiting for %s. '
 255                           'Attempting to recover.', str(device))
 256
 257       try:
 258         device.Reboot()
 259         device.WaitUntilFullyBooted()
 260       except device_errors.CommandFailedError:
 261         logging.exception('Failure while waiting for %s.', str(device))
 262         if blacklist:
 263           blacklist.Extend([device.adb.GetDeviceSerial()])
 264       except device_errors.CommandTimeoutError:
 265         logging.exception('Timed out while waiting for %s. ', str(device))
 266         if blacklist:
 267           blacklist.Extend([device.adb.GetDeviceSerial()])
 268
 269   device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)
 270
 271
 272 def main():
 273   parser = argparse.ArgumentParser()
 274   parser.add_argument('--out-dir',
 275                       help='Directory where the device path is stored',
 276                       default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 277   parser.add_argument('--restart-usb', action='store_true',
 278                       help='DEPRECATED. '
 279                            'This script now always tries to reset USB.')
 280   parser.add_argument('--json-output',
 281                       help='Output JSON information into a specified file.')
 282   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
 283   parser.add_argument('-v', '--verbose', action='count', default=1,
 284                       help='Log more information.')
 285
 286   args = parser.parse_args()
 287
 288   run_tests_helper.SetLogLevel(args.verbose)
 289
 290   blacklist = (device_blacklist.Blacklist(args.blacklist_file)
 291                if args.blacklist_file
 292                else None)
 293
 294   last_devices_path = os.path.join(
 295       args.out_dir, device_list.LAST_DEVICES_FILENAME)
 296   try:
 297     expected_devices = set(
 298         device_list.GetPersistentDeviceList(last_devices_path))
 299   except IOError:
 300     expected_devices = set()
 301
 302   usb_devices = set(lsusb.get_android_devices())
 303   devices = [device_utils.DeviceUtils(s)
 304              for s in expected_devices.union(usb_devices)]
 305
 306   RecoverDevices(devices, blacklist)
 307   statuses = DeviceStatus(devices, blacklist)
 308
 309   # Log the state of all devices.
 310   for status in statuses:
 311     logging.info(status['serial'])
 312     adb_status = status.get('adb_status')
 313     blacklisted = status.get('blacklisted')
 314     logging.info('  USB status: %s',
 315                  'online' if status.get('usb_status') else 'offline')
 316     logging.info('  ADB status: %s', adb_status)
 317     logging.info('  Blacklisted: %s', str(blacklisted))
 318     if adb_status == 'device' and not blacklisted:
 319       logging.info('  Device type: %s', status.get('ro.build.product'))
 320       logging.info('  OS build: %s', status.get('ro.build.id'))
 321       logging.info('  OS build fingerprint: %s',
 322                    status.get('ro.build.fingerprint'))
 323       logging.info('  Battery state:')
 324       for k, v in status.get('battery', {}).iteritems():
 325         logging.info('    %s: %s', k, v)
 326       logging.info('  IMEI slice: %s', status.get('imei_slice'))
 327       logging.info('  WiFi IP: %s', status.get('wifi_ip'))
 328
 329   # Update the last devices file.
 330   device_list.WritePersistentDeviceList(
 331       last_devices_path,
 332       [status['serial'] for status in statuses])
 333
 334   # Write device info to file for buildbot info display.
 335   if os.path.exists('/home/chrome-bot'):
 336     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 337       for status in statuses:
 338         try:
 339           if status['adb_status'] == 'device':
 340             f.write('{serial} {adb_status} {build_product} {build_id} '
 341                     '{temperature:.1f}C {level}%\n'.format(
 342                 serial=status['serial'],
 343                 adb_status=status['adb_status'],
 344                 build_product=status['type'],
 345                 build_id=status['build'],
 346                 temperature=float(status['battery']['temperature']) / 10,
 347                 level=status['battery']['level']
 348             ))
 349           else:
 350             f.write('{serial} {adb_status}'.format(
 351                 serial=status['serial'],
 352                 adb_status=status['adb_status']
 353             ))
 354         except Exception: # pylint: disable=broad-except
 355           pass
 356
 357   # Dump the device statuses to JSON.
 358   if args.json_output:
 359     with open(args.json_output, 'wb') as f:
 360       f.write(json.dumps(statuses, indent=4))
 361
 362   live_devices = [status['serial'] for status in statuses
 363                   if (status['adb_status'] == 'device'
 364                       and not _IsBlacklisted(status['serial'], blacklist))]
 365
 366   # If all devices failed, or if there are no devices, it's an infra error.
 367   return 0 if live_devices else constants.INFRA_EXIT_CODE
 368
 369
 370 if __name__ == '__main__':
 371   sys.exit(main())