build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
  30 from devil.utils import reset_usb
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import adb_wrapper
  34 from pylib.device import battery_utils
  35 from pylib.device import device_blacklist
  36 from pylib.device import device_errors
  37 from pylib.device import device_list
  38 from pylib.device import device_utils
  39 from pylib.utils import run_tests_helper
  40 from pylib.utils import timeout_retry
  41
  42 _RE_DEVICE_ID = re.compile('Device ID = (\d+)')
  43
  44 def DeviceInfo(device, options):
  45   """Gathers info on a device via various adb calls.
  46
  47   Args:
  48     device: A DeviceUtils instance for the device to construct info about.
  49
  50   Returns:
  51     Tuple of device type, build id, report as a string, error messages, and
  52     boolean indicating whether or not device can be used for testing.
  53   """
  54   battery = battery_utils.BatteryUtils(device)
  55
  56   build_product = ''
  57   build_id = ''
  58   battery_level = 100
  59   errors = []
  60   dev_good = True
  61   json_data = {}
  62
  63   try:
  64     build_product = device.build_product
  65     build_id = device.build_id
  66
  67     json_data = {
  68       'serial': device.adb.GetDeviceSerial(),
  69       'type': build_product,
  70       'build': build_id,
  71       'build_detail': device.GetProp('ro.build.fingerprint'),
  72       'battery': {},
  73       'imei_slice': 'Unknown',
  74       'wifi_ip': device.GetProp('dhcp.wlan0.ipaddress'),
  75     }
  76
  77     battery_info = {}
  78     try:
  79       battery_info = battery.GetBatteryInfo(timeout=5)
  80       battery_level = int(battery_info.get('level', battery_level))
  81       json_data['battery'] = battery_info
  82     except device_errors.CommandFailedError:
  83       logging.exception('Failed to get battery information for %s', str(device))
  84
  85     try:
  86       for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  87                                       check_return=True, timeout=5):
  88         m = _RE_DEVICE_ID.match(l)
  89         if m:
  90           json_data['imei_slice'] = m.group(1)[-6:]
  91     except device_errors.CommandFailedError:
  92       logging.exception('Failed to get IMEI slice for %s', str(device))
  93
  94     if battery_level < 15:
  95       errors += ['Device critically low in battery.']
  96       dev_good = False
  97       if not battery.GetCharging():
  98         battery.SetCharging(True)
  99     if not options.no_provisioning_check:
 100       setup_wizard_disabled = (
 101           device.GetProp('ro.setupwizard.mode') == 'DISABLED')
 102       if not setup_wizard_disabled and device.build_type != 'user':
 103         errors += ['Setup wizard not disabled. Was it provisioned correctly?']
 104     if (device.product_name == 'mantaray' and
 105         battery_info.get('AC powered', None) != 'true'):
 106       errors += ['Mantaray device not connected to AC power.']
 107   except device_errors.CommandFailedError:
 108     logging.exception('Failure while getting device status.')
 109     dev_good = False
 110   except device_errors.CommandTimeoutError:
 111     logging.exception('Timeout while getting device status.')
 112     dev_good = False
 113
 114   return (build_product, build_id, battery_level, errors, dev_good, json_data)
 115
 116
 117 def CheckForMissingDevices(options, devices):
 118   """Uses file of previous online devices to detect broken phones.
 119
 120   Args:
 121     options: out_dir parameter of options argument is used as the base
 122       directory to load and update the cache file.
 123     devices: A list of DeviceUtils instance for the currently visible and
 124       online attached devices.
 125   """
 126   out_dir = os.path.abspath(options.out_dir)
 127   device_serials = set(d.adb.GetDeviceSerial() for d in devices)
 128
 129   # last_devices denotes all known devices prior to this run
 130   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 131   last_missing_devices_path = os.path.join(out_dir,
 132       device_list.LAST_MISSING_DEVICES_FILENAME)
 133   try:
 134     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 135   except IOError:
 136     # Ignore error, file might not exist
 137     last_devices = []
 138
 139   try:
 140     last_missing_devices = device_list.GetPersistentDeviceList(
 141         last_missing_devices_path)
 142   except IOError:
 143     last_missing_devices = []
 144
 145   missing_devs = list(set(last_devices) - device_serials)
 146   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 147
 148   buildbot_slavename = os.environ.get('BUILDBOT_SLAVENAME')
 149   buildbot_buildername = os.environ.get('BUILDBOT_BUILDERNAME')
 150   buildbot_buildnumber = os.environ.get('BUILDBOT_BUILDNUMBER')
 151
 152   if new_missing_devs and buildbot_slavename:
 153     logging.info('new_missing_devs %s' % new_missing_devs)
 154     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 155     bb_annotations.PrintSummaryText(devices_missing_msg)
 156
 157     from_address = 'chrome-bot@chromium.org'
 158     to_addresses = ['chrome-labs-tech-ticket@google.com',
 159                     'chrome-android-device-alert@google.com']
 160     cc_addresses = ['chrome-android-device-alert@google.com']
 161     subject = 'Devices offline on %s, %s, %s' % (
 162         buildbot_slavename, buildbot_buildername, buildbot_buildnumber)
 163     msg = ('Please reboot the following devices:\n%s' %
 164            '\n'.join(map(str, new_missing_devs)))
 165     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 166
 167   unauthorized_devices = adb_wrapper.AdbWrapper.Devices(
 168       desired_state='unauthorized')
 169   if unauthorized_devices:
 170     logging.info('unauthorized devices:')
 171     for ud in unauthorized_devices:
 172       logging.info('  %s', ud)
 173
 174     if buildbot_slavename:
 175       from_address = 'chrome-bot@chromium.org'
 176       to_addresses = [
 177           'jbudorick@chromium.org',
 178           'chrome-android-device-alert@google.com']
 179       subject = 'Unauthorized devices on %s' % buildbot_slavename
 180       msg = ['The following devices are offline on %s' % buildbot_slavename]
 181       msg += ['  %s' % ud for ud in unauthorized_devices]
 182       msg += ['', 'Detected on %s #%s' % (buildbot_buildername,
 183                                           buildbot_buildnumber)]
 184       msg = '\n'.join(msg)
 185       SendEmail(from_address, to_addresses, [], subject, msg)
 186
 187   all_known_devices = list(device_serials | set(last_devices))
 188   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 189   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 190
 191   if not all_known_devices:
 192     # This can happen if for some reason the .last_devices file is not
 193     # present or if it was empty.
 194     return ['No online devices. Have any devices been plugged in?']
 195   if missing_devs:
 196     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 197     bb_annotations.PrintSummaryText(devices_missing_msg)
 198     return ['Current online devices: %s' % ', '.join(d for d in device_serials),
 199             '%s are no longer visible. Were they removed?' % missing_devs]
 200   else:
 201     new_devs = device_serials - set(last_devices)
 202     if new_devs and os.path.exists(last_devices_path):
 203       bb_annotations.PrintWarning()
 204       bb_annotations.PrintSummaryText(
 205           '%d new devices detected' % len(new_devs))
 206       logging.info('New devices detected:')
 207       for d in new_devs:
 208         logging.info('  %s', d)
 209
 210
 211 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 212   # TODO(jbudorick): Transition from email alerts for every failure to a more
 213   # sustainable solution.
 214   msg_body = '\r\n'.join(['From: %s' % from_address,
 215                           'To: %s' % ', '.join(to_addresses),
 216                           'CC: %s' % ', '.join(cc_addresses),
 217                           'Subject: %s' % subject, '', msg])
 218   try:
 219     server = smtplib.SMTP('localhost')
 220     server.sendmail(from_address, to_addresses, msg_body)
 221     server.quit()
 222   except Exception:
 223     logging.exception('Failed to send alert email.')
 224
 225
 226 def KillAllAdb():
 227   def GetAllAdb():
 228     for p in psutil.process_iter():
 229       try:
 230         if 'adb' in p.name:
 231           yield p
 232       except (psutil.NoSuchProcess, psutil.AccessDenied):
 233         pass
 234
 235   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 236     for p in GetAllAdb():
 237       try:
 238         logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
 239                      ' '.join(p.cmdline))
 240         p.send_signal(sig)
 241       except (psutil.NoSuchProcess, psutil.AccessDenied):
 242         pass
 243   for p in GetAllAdb():
 244     try:
 245       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
 246                     ' '.join(p.cmdline))
 247     except (psutil.NoSuchProcess, psutil.AccessDenied):
 248       pass
 249
 250
 251 def RecoverDevices(args):
 252   # Remove the last build's "bad devices" before checking device statuses.
 253   device_blacklist.ResetBlacklist()
 254
 255   previous_devices = set(a.GetDeviceSerial()
 256                          for a in adb_wrapper.AdbWrapper.Devices())
 257
 258   KillAllAdb()
 259   reset_usb.reset_all_android_devices()
 260
 261   try:
 262     expected_devices = set(device_list.GetPersistentDeviceList(
 263         os.path.join(args.out_dir, device_list.LAST_DEVICES_FILENAME)))
 264   except IOError:
 265     expected_devices = set()
 266
 267   all_devices = [device_utils.DeviceUtils(d)
 268                  for d in previous_devices.union(expected_devices)]
 269
 270   def blacklisting_recovery(device):
 271     try:
 272       device.WaitUntilFullyBooted()
 273     except device_errors.CommandFailedError:
 274       logging.exception('Failure while waiting for %s. Adding to blacklist.',
 275                         str(device))
 276       device_blacklist.ExtendBlacklist([str(device)])
 277     except device_errors.CommandTimeoutError:
 278       logging.exception('Timed out while waiting for %s. Adding to blacklist.',
 279                         str(device))
 280       device_blacklist.ExtendBlacklist([str(device)])
 281
 282   device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
 283
 284   devices = device_utils.DeviceUtils.HealthyDevices()
 285   device_serials = set(d.adb.GetDeviceSerial() for d in devices)
 286
 287   missing_devices = expected_devices.difference(device_serials)
 288   new_devices = device_serials.difference(expected_devices)
 289
 290   if missing_devices or new_devices:
 291     logging.warning('expected_devices:')
 292     for d in sorted(expected_devices):
 293       logging.warning('  %s', d)
 294     logging.warning('devices:')
 295     for d in sorted(device_serials):
 296       logging.warning('  %s', d)
 297
 298   return devices
 299
 300
 301 def main():
 302   parser = optparse.OptionParser()
 303   parser.add_option('', '--out-dir',
 304                     help='Directory where the device path is stored',
 305                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 306   parser.add_option('--no-provisioning-check', action='store_true',
 307                     help='Will not check if devices are provisioned properly.')
 308   parser.add_option('--device-status-dashboard', action='store_true',
 309                     help='Output device status data for dashboard.')
 310   parser.add_option('--restart-usb', action='store_true',
 311                     help='DEPRECATED. '
 312                          'This script now always tries to reset USB.')
 313   parser.add_option('--json-output',
 314                     help='Output JSON information into a specified file.')
 315   parser.add_option('-v', '--verbose', action='count', default=1,
 316                     help='Log more information.')
 317
 318   options, args = parser.parse_args()
 319   if args:
 320     parser.error('Unknown options %s' % args)
 321
 322   run_tests_helper.SetLogLevel(options.verbose)
 323
 324   devices = RecoverDevices(options)
 325
 326   types, builds, batteries, errors, devices_ok, json_data = (
 327       [], [], [], [], [], [])
 328   if devices:
 329     types, builds, batteries, errors, devices_ok, json_data = (
 330         zip(*[DeviceInfo(dev, options) for dev in devices]))
 331
 332   # Write device info to file for buildbot info display.
 333   if os.path.exists('/home/chrome-bot'):
 334     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 335       for device in json_data:
 336         try:
 337           f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
 338               device['build'], float(device['battery']['temperature']) / 10,
 339               device['battery']['level']))
 340         except Exception:
 341           pass
 342
 343   err_msg = CheckForMissingDevices(options, devices) or []
 344
 345   unique_types = list(set(types))
 346   unique_builds = list(set(builds))
 347
 348   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 349                            % (len(devices), unique_types, unique_builds))
 350
 351   for j in json_data:
 352     logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
 353     logging.info('  Build: %s (%s)', j.get('build'), j.get('build_detail'))
 354     logging.info('  Current Battery Service state:')
 355     for k, v in j.get('battery', {}).iteritems():
 356       logging.info('    %s: %s', k, v)
 357     logging.info('  IMEI slice: %s', j.get('imei_slice'))
 358     logging.info('  WiFi IP: %s', j.get('wifi_ip'))
 359
 360
 361   for dev, dev_errors in zip(devices, errors):
 362     if dev_errors:
 363       err_msg += ['%s errors:' % str(dev)]
 364       err_msg += ['    %s' % error for error in dev_errors]
 365
 366   if err_msg:
 367     bb_annotations.PrintWarning()
 368     for e in err_msg:
 369       logging.error(e)
 370     from_address = 'buildbot@chromium.org'
 371     to_addresses = ['chromium-android-device-alerts@google.com']
 372     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 373     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 374     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 375     SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
 376
 377   if options.device_status_dashboard:
 378     offline_devices = [
 379         device_utils.DeviceUtils(a)
 380         for a in adb_wrapper.AdbWrapper.Devices(desired_state=None)
 381         if a.GetState() == 'offline']
 382
 383     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 384                                               [len(devices)], 'devices')
 385     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 386                                               [len(offline_devices)], 'devices',
 387                                               'unimportant')
 388     for dev, battery in zip(devices, batteries):
 389       perf_tests_results_helper.PrintPerfResult('DeviceBattery', str(dev),
 390                                                 [battery], '%',
 391                                                 'unimportant')
 392
 393   if options.json_output:
 394     with open(options.json_output, 'wb') as f:
 395       f.write(json.dumps(json_data, indent=4))
 396
 397   num_failed_devs = 0
 398   for device_ok, device in zip(devices_ok, devices):
 399     if not device_ok:
 400       logging.warning('Blacklisting %s', str(device))
 401       device_blacklist.ExtendBlacklist([str(device)])
 402       num_failed_devs += 1
 403
 404   if num_failed_devs == len(devices):
 405     return 2
 406
 407   if not devices:
 408     return 1
 409
 410
 411 if __name__ == '__main__':
 412   sys.exit(main())