build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import constants
  31 from pylib.cmd_helper import GetCmdOutput
  32 from pylib.device import adb_wrapper
  33 from pylib.device import battery_utils
  34 from pylib.device import device_blacklist
  35 from pylib.device import device_errors
  36 from pylib.device import device_list
  37 from pylib.device import device_utils
  38 from pylib.utils import run_tests_helper
  39
  40 _RE_DEVICE_ID = re.compile('Device ID = (\d+)')
  41
  42 def DeviceInfo(device, options):
  43   """Gathers info on a device via various adb calls.
  44
  45   Args:
  46     device: A DeviceUtils instance for the device to construct info about.
  47
  48   Returns:
  49     Tuple of device type, build id, report as a string, error messages, and
  50     boolean indicating whether or not device can be used for testing.
  51   """
  52   battery = battery_utils.BatteryUtils(device)
  53
  54   build_product = ''
  55   build_id = ''
  56   battery_level = 100
  57   errors = []
  58   dev_good = True
  59   json_data = {}
  60
  61   try:
  62     build_product = device.build_product
  63     build_id = device.build_id
  64
  65     json_data = {
  66       'serial': device.adb.GetDeviceSerial(),
  67       'type': build_product,
  68       'build': build_id,
  69       'build_detail': device.GetProp('ro.build.fingerprint'),
  70       'battery': {},
  71       'imei_slice': 'Unknown',
  72       'wifi_ip': device.GetProp('dhcp.wlan0.ipaddress'),
  73     }
  74
  75     battery_info = {}
  76     try:
  77       battery_info = battery.GetBatteryInfo(timeout=5)
  78       battery_level = int(battery_info.get('level', battery_level))
  79       json_data['battery'] = battery_info
  80     except device_errors.CommandFailedError:
  81       logging.exception('Failed to get battery information for %s', str(device))
  82
  83     try:
  84       for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  85                                       check_return=True, timeout=5):
  86         m = _RE_DEVICE_ID.match(l)
  87         if m:
  88           json_data['imei_slice'] = m.group(1)[-6:]
  89     except device_errors.CommandFailedError:
  90       logging.exception('Failed to get IMEI slice for %s', str(device))
  91
  92     if battery_level < 15:
  93       errors += ['Device critically low in battery.']
  94       dev_good = False
  95       if not battery.GetCharging():
  96         battery.SetCharging(True)
  97     if not options.no_provisioning_check:
  98       setup_wizard_disabled = (
  99           device.GetProp('ro.setupwizard.mode') == 'DISABLED')
 100       if not setup_wizard_disabled and device.build_type != 'user':
 101         errors += ['Setup wizard not disabled. Was it provisioned correctly?']
 102     if (device.product_name == 'mantaray' and
 103         battery_info.get('AC powered', None) != 'true'):
 104       errors += ['Mantaray device not connected to AC power.']
 105   except device_errors.CommandFailedError:
 106     logging.exception('Failure while getting device status.')
 107     dev_good = False
 108   except device_errors.CommandTimeoutError:
 109     logging.exception('Timeout while getting device status.')
 110     dev_good = False
 111
 112   return (build_product, build_id, battery_level, errors, dev_good, json_data)
 113
 114
 115 def CheckForMissingDevices(options, devices):
 116   """Uses file of previous online devices to detect broken phones.
 117
 118   Args:
 119     options: out_dir parameter of options argument is used as the base
 120       directory to load and update the cache file.
 121     devices: A list of DeviceUtils instance for the currently visible and
 122       online attached devices.
 123   """
 124   out_dir = os.path.abspath(options.out_dir)
 125   device_serials = set(d.adb.GetDeviceSerial() for d in devices)
 126
 127   # last_devices denotes all known devices prior to this run
 128   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 129   last_missing_devices_path = os.path.join(out_dir,
 130       device_list.LAST_MISSING_DEVICES_FILENAME)
 131   try:
 132     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 133   except IOError:
 134     # Ignore error, file might not exist
 135     last_devices = []
 136
 137   try:
 138     last_missing_devices = device_list.GetPersistentDeviceList(
 139         last_missing_devices_path)
 140   except IOError:
 141     last_missing_devices = []
 142
 143   missing_devs = list(set(last_devices) - device_serials)
 144   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 145
 146   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 147     logging.info('new_missing_devs %s' % new_missing_devs)
 148     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 149     bb_annotations.PrintSummaryText(devices_missing_msg)
 150
 151     from_address = 'chrome-bot@chromium.org'
 152     to_addresses = ['chrome-labs-tech-ticket@google.com',
 153                     'chrome-android-device-alert@google.com']
 154     cc_addresses = ['chrome-android-device-alert@google.com']
 155     subject = 'Devices offline on %s, %s, %s' % (
 156       os.environ.get('BUILDBOT_SLAVENAME'),
 157       os.environ.get('BUILDBOT_BUILDERNAME'),
 158       os.environ.get('BUILDBOT_BUILDNUMBER'))
 159     msg = ('Please reboot the following devices:\n%s' %
 160            '\n'.join(map(str, new_missing_devs)))
 161     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 162
 163   all_known_devices = list(device_serials | set(last_devices))
 164   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 165   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 166
 167   if not all_known_devices:
 168     # This can happen if for some reason the .last_devices file is not
 169     # present or if it was empty.
 170     return ['No online devices. Have any devices been plugged in?']
 171   if missing_devs:
 172     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 173     bb_annotations.PrintSummaryText(devices_missing_msg)
 174     return ['Current online devices: %s' % ', '.join(d for d in device_serials),
 175             '%s are no longer visible. Were they removed?' % missing_devs]
 176   else:
 177     new_devs = device_serials - set(last_devices)
 178     if new_devs and os.path.exists(last_devices_path):
 179       bb_annotations.PrintWarning()
 180       bb_annotations.PrintSummaryText(
 181           '%d new devices detected' % len(new_devs))
 182       logging.info('New devices detected:')
 183       for d in new_devs:
 184         logging.info('  %s', d)
 185
 186
 187 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 188   msg_body = '\r\n'.join(['From: %s' % from_address,
 189                           'To: %s' % ', '.join(to_addresses),
 190                           'CC: %s' % ', '.join(cc_addresses),
 191                           'Subject: %s' % subject, '', msg])
 192   try:
 193     server = smtplib.SMTP('localhost')
 194     server.sendmail(from_address, to_addresses, msg_body)
 195     server.quit()
 196   except Exception:
 197     logging.exception('Failed to send alert email.')
 198
 199
 200 def RestartUsb():
 201   if not os.path.isfile('/usr/bin/restart_usb'):
 202     logging.error('Could not restart usb. ''/usr/bin/restart_usb not '
 203                   'installed on host (see BUG=305769).')
 204     return False
 205
 206   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 207   lsusb_output, _ = lsusb_proc.communicate()
 208   if lsusb_proc.returncode:
 209     logging.error('Could not get list of USB ports (i.e. lsusb).')
 210     return lsusb_proc.returncode
 211
 212   usb_devices = [re.findall(r'Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 213                  for lsusb_line in lsusb_output.strip().split('\n')]
 214
 215   all_restarted = True
 216   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 217   # connection. If a parent node (e.g. usb hub) is restarted before the
 218   # devices connected to it, the (bus, dev) for the hub can change, making the
 219   # output we have wrong. This way we restart the devices before the hub.
 220   for (bus, dev) in reversed(sorted(usb_devices)):
 221     # Can not restart root usb connections
 222     if dev != '001':
 223       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 224       if return_code:
 225         logging.error('Error restarting USB device /dev/bus/usb/%s/%s',
 226                       bus, dev)
 227         all_restarted = False
 228       else:
 229         logging.info('Restarted USB device /dev/bus/usb/%s/%s', bus, dev)
 230
 231   return all_restarted
 232
 233
 234 def KillAllAdb():
 235   def GetAllAdb():
 236     for p in psutil.process_iter():
 237       try:
 238         if 'adb' in p.name:
 239           yield p
 240       except (psutil.NoSuchProcess, psutil.AccessDenied):
 241         pass
 242
 243   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 244     for p in GetAllAdb():
 245       try:
 246         logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
 247                      ' '.join(p.cmdline))
 248         p.send_signal(sig)
 249       except (psutil.NoSuchProcess, psutil.AccessDenied):
 250         pass
 251   for p in GetAllAdb():
 252     try:
 253       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
 254                     ' '.join(p.cmdline))
 255     except (psutil.NoSuchProcess, psutil.AccessDenied):
 256       pass
 257
 258
 259 def main():
 260   parser = optparse.OptionParser()
 261   parser.add_option('', '--out-dir',
 262                     help='Directory where the device path is stored',
 263                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 264   parser.add_option('--no-provisioning-check', action='store_true',
 265                     help='Will not check if devices are provisioned properly.')
 266   parser.add_option('--device-status-dashboard', action='store_true',
 267                     help='Output device status data for dashboard.')
 268   parser.add_option('--restart-usb', action='store_true',
 269                     help='Restart USB ports before running device check.')
 270   parser.add_option('--json-output',
 271                     help='Output JSON information into a specified file.')
 272   parser.add_option('-v', '--verbose', action='count', default=1,
 273                     help='Log more information.')
 274
 275   options, args = parser.parse_args()
 276   if args:
 277     parser.error('Unknown options %s' % args)
 278
 279   run_tests_helper.SetLogLevel(options.verbose)
 280
 281   # Remove the last build's "bad devices" before checking device statuses.
 282   device_blacklist.ResetBlacklist()
 283
 284   try:
 285     expected_devices = device_list.GetPersistentDeviceList(
 286         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 287   except IOError:
 288     expected_devices = []
 289   devices = device_utils.DeviceUtils.HealthyDevices()
 290   device_serials = [d.adb.GetDeviceSerial() for d in devices]
 291   # Only restart usb if devices are missing.
 292   if set(expected_devices) != set(device_serials):
 293     logging.warning('expected_devices: %s', expected_devices)
 294     logging.warning('devices: %s', device_serials)
 295     KillAllAdb()
 296     retries = 5
 297     usb_restarted = True
 298     if options.restart_usb:
 299       if not RestartUsb():
 300         usb_restarted = False
 301         bb_annotations.PrintWarning()
 302         logging.error('USB reset stage failed, '
 303                       'wait for any device to come back.')
 304     while retries:
 305       logging.info('retry adb devices...')
 306       time.sleep(1)
 307       devices = device_utils.DeviceUtils.HealthyDevices()
 308       device_serials = [d.adb.GetDeviceSerial() for d in devices]
 309       if set(expected_devices) == set(device_serials):
 310         # All devices are online, keep going.
 311         break
 312       if not usb_restarted and devices:
 313         # The USB wasn't restarted, but there's at least one device online.
 314         # No point in trying to wait for all devices.
 315         break
 316       retries -= 1
 317
 318   types, builds, batteries, errors, devices_ok, json_data = (
 319       [], [], [], [], [], [])
 320   if devices:
 321     types, builds, batteries, errors, devices_ok, json_data = (
 322         zip(*[DeviceInfo(dev, options) for dev in devices]))
 323
 324   # Write device info to file for buildbot info display.
 325   if os.path.exists('/home/chrome-bot'):
 326     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 327       for device in json_data:
 328         try:
 329           f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
 330               device['build'], float(device['battery']['temperature']) / 10,
 331               device['battery']['level']))
 332         except Exception:
 333           pass
 334
 335   err_msg = CheckForMissingDevices(options, devices) or []
 336
 337   unique_types = list(set(types))
 338   unique_builds = list(set(builds))
 339
 340   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 341                            % (len(devices), unique_types, unique_builds))
 342
 343   for j in json_data:
 344     logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
 345     logging.info('  Build: %s (%s)', j.get('build'), j.get('build_detail'))
 346     logging.info('  Current Battery Service state:')
 347     for k, v in j.get('battery', {}).iteritems():
 348       logging.info('    %s: %s', k, v)
 349     logging.info('  IMEI slice: %s', j.get('imei_slice'))
 350     logging.info('  WiFi IP: %s', j.get('wifi_ip'))
 351
 352
 353   for dev, dev_errors in zip(devices, errors):
 354     if dev_errors:
 355       err_msg += ['%s errors:' % str(dev)]
 356       err_msg += ['    %s' % error for error in dev_errors]
 357
 358   if err_msg:
 359     bb_annotations.PrintWarning()
 360     for e in err_msg:
 361       logging.error(e)
 362     from_address = 'buildbot@chromium.org'
 363     to_addresses = ['chromium-android-device-alerts@google.com']
 364     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 365     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 366     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 367     SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
 368
 369   if options.device_status_dashboard:
 370     offline_devices = [
 371         device_utils.DeviceUtils(a)
 372         for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
 373         if a.GetState() == 'offline']
 374
 375     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 376                                               [len(devices)], 'devices')
 377     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 378                                               [len(offline_devices)], 'devices',
 379                                               'unimportant')
 380     for dev, battery in zip(devices, batteries):
 381       perf_tests_results_helper.PrintPerfResult('DeviceBattery', str(dev),
 382                                                 [battery], '%',
 383                                                 'unimportant')
 384
 385   if options.json_output:
 386     with open(options.json_output, 'wb') as f:
 387       f.write(json.dumps(json_data, indent=4))
 388
 389   num_failed_devs = 0
 390   for device_ok, device in zip(devices_ok, devices):
 391     if not device_ok:
 392       logging.warning('Blacklisting %s', str(device))
 393       device_blacklist.ExtendBlacklist([str(device)])
 394       num_failed_devs += 1
 395
 396   if num_failed_devs == len(devices):
 397     return 2
 398
 399   if not devices:
 400     return 1
 401
 402
 403 if __name__ == '__main__':
 404   sys.exit(main())