build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import android_commands
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import battery_utils
  34 from pylib.device import device_blacklist
  35 from pylib.device import device_errors
  36 from pylib.device import device_list
  37 from pylib.device import device_utils
  38
  39 _RE_DEVICE_ID = re.compile('Device ID = (\d+)')
  40
  41 def DeviceInfo(serial, options):
  42   """Gathers info on a device via various adb calls.
  43
  44   Args:
  45     serial: The serial of the attached device to construct info about.
  46
  47   Returns:
  48     Tuple of device type, build id, report as a string, error messages, and
  49     boolean indicating whether or not device can be used for testing.
  50   """
  51   device = device_utils.DeviceUtils(serial)
  52   battery = battery_utils.BatteryUtils(device)
  53
  54   battery_info = {}
  55   try:
  56     battery_info = battery.GetBatteryInfo()
  57   except Exception as e:
  58     battery_info = {}
  59     logging.error('Unable to obtain battery info for %s, %s', serial, e)
  60
  61   battery_level = int(battery_info.get('level', 100))
  62
  63   imei_slice = 'Unknown'
  64   try:
  65     for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  66                                     check_return=True):
  67       m = _RE_DEVICE_ID.match(l)
  68       if m:
  69         imei_slice = m.group(1)[-6:]
  70   except device_errors.CommandFailedError:
  71     logging.exception('Failed to get IMEI slice.')
  72   except device_errors.CommandTimeoutError:
  73     logging.exception('Timed out while attempting to get IMEI slice.')
  74
  75   json_data = {
  76     'serial': serial,
  77     'type': device.build_product,
  78     'build': device.build_id,
  79     'build_detail': device.GetProp('ro.build.fingerprint'),
  80     'battery': battery_info,
  81     'imei_slice': imei_slice,
  82     'wifi_ip': device.GetProp('dhcp.wlan0.ipaddress'),
  83   }
  84   report = [
  85     'Device %s (%s)' % (str(device), device.build_product),
  86     '  Build: %s (%s)' % (device.build_id, json_data['build_detail']),
  87     '  Current Battery Service state: ',
  88     '\n'.join('    %s: %s' % (k, v) for k, v in battery_info.iteritems()),
  89     '  IMEI slice: %s' % imei_slice,
  90     '  Wifi IP: %s' % json_data['wifi_ip'],
  91     ''
  92   ]
  93
  94   errors = []
  95   dev_good = True
  96   if battery_level < 15:
  97     errors += ['Device critically low in battery. Will add to blacklist.']
  98     dev_good = False
  99     if not battery.GetCharging():
 100       try:
 101         battery.SetCharging(True)
 102       except device_errors.CommandFailedError:
 103         logging.exception('Device %s is not charging', str(device))
 104   if not options.no_provisioning_check:
 105     setup_wizard_disabled = (
 106         device.GetProp('ro.setupwizard.mode') == 'DISABLED')
 107     if not setup_wizard_disabled and device.build_type != 'user':
 108       errors += ['Setup wizard not disabled. Was it provisioned correctly?']
 109   if (device.product_name == 'mantaray' and
 110       battery_info.get('AC powered', None) != 'true'):
 111     errors += ['Mantaray device not connected to AC power.']
 112
 113   full_report = '\n'.join(report)
 114
 115   return (device.build_product, device.build_id, battery_level, full_report,
 116           errors, dev_good, json_data)
 117
 118
 119 def CheckForMissingDevices(options, adb_online_devs):
 120   """Uses file of previous online devices to detect broken phones.
 121
 122   Args:
 123     options: out_dir parameter of options argument is used as the base
 124              directory to load and update the cache file.
 125     adb_online_devs: A list of serial numbers of the currently visible
 126                      and online attached devices.
 127   """
 128   # TODO(navabi): remove this once the bug that causes different number
 129   # of devices to be detected between calls is fixed.
 130   logger = logging.getLogger()
 131   logger.setLevel(logging.INFO)
 132
 133   out_dir = os.path.abspath(options.out_dir)
 134
 135   # last_devices denotes all known devices prior to this run
 136   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 137   last_missing_devices_path = os.path.join(out_dir,
 138       device_list.LAST_MISSING_DEVICES_FILENAME)
 139   try:
 140     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 141   except IOError:
 142     # Ignore error, file might not exist
 143     last_devices = []
 144
 145   try:
 146     last_missing_devices = device_list.GetPersistentDeviceList(
 147         last_missing_devices_path)
 148   except IOError:
 149     last_missing_devices = []
 150
 151   missing_devs = list(set(last_devices) - set(adb_online_devs))
 152   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 153
 154   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 155     logging.info('new_missing_devs %s' % new_missing_devs)
 156     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 157     bb_annotations.PrintSummaryText(devices_missing_msg)
 158
 159     from_address = 'chrome-bot@chromium.org'
 160     to_addresses = ['chrome-labs-tech-ticket@google.com',
 161                     'chrome-android-device-alert@google.com']
 162     cc_addresses = ['chrome-android-device-alert@google.com']
 163     subject = 'Devices offline on %s, %s, %s' % (
 164       os.environ.get('BUILDBOT_SLAVENAME'),
 165       os.environ.get('BUILDBOT_BUILDERNAME'),
 166       os.environ.get('BUILDBOT_BUILDNUMBER'))
 167     msg = ('Please reboot the following devices:\n%s' %
 168            '\n'.join(map(str, new_missing_devs)))
 169     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 170
 171   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 172   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 173   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 174
 175   if not all_known_devices:
 176     # This can happen if for some reason the .last_devices file is not
 177     # present or if it was empty.
 178     return ['No online devices. Have any devices been plugged in?']
 179   if missing_devs:
 180     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 181     bb_annotations.PrintSummaryText(devices_missing_msg)
 182
 183     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 184     # GetAttachedDevices to compare results.
 185     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 186                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 187                   (urllib.quote('Device Offline'),
 188                    urllib.quote('Buildbot: %s %s\n'
 189                                 'Build: %s\n'
 190                                 '(please don\'t change any labels)' %
 191                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 192                                  os.environ.get('BUILDBOT_SLAVENAME'),
 193                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 194     return ['Current online devices: %s' % adb_online_devs,
 195             '%s are no longer visible. Were they removed?\n' % missing_devs,
 196             'SHERIFF:\n',
 197             '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
 198             'Cache file: %s\n\n' % last_devices_path,
 199             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 200             'adb devices(GetAttachedDevices): %s' % adb_online_devs]
 201   else:
 202     new_devs = set(adb_online_devs) - set(last_devices)
 203     if new_devs and os.path.exists(last_devices_path):
 204       bb_annotations.PrintWarning()
 205       bb_annotations.PrintSummaryText(
 206           '%d new devices detected' % len(new_devs))
 207       print ('New devices detected %s. And now back to your '
 208              'regularly scheduled program.' % list(new_devs))
 209
 210
 211 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 212   msg_body = '\r\n'.join(['From: %s' % from_address,
 213                           'To: %s' % ', '.join(to_addresses),
 214                           'CC: %s' % ', '.join(cc_addresses),
 215                           'Subject: %s' % subject, '', msg])
 216   try:
 217     server = smtplib.SMTP('localhost')
 218     server.sendmail(from_address, to_addresses, msg_body)
 219     server.quit()
 220   except Exception as e:
 221     print 'Failed to send alert email. Error: %s' % e
 222
 223
 224 def RestartUsb():
 225   if not os.path.isfile('/usr/bin/restart_usb'):
 226     print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
 227            'on host (see BUG=305769).')
 228     return False
 229
 230   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 231   lsusb_output, _ = lsusb_proc.communicate()
 232   if lsusb_proc.returncode:
 233     print 'Error: Could not get list of USB ports (i.e. lsusb).'
 234     return lsusb_proc.returncode
 235
 236   usb_devices = [re.findall(r'Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 237                  for lsusb_line in lsusb_output.strip().split('\n')]
 238
 239   all_restarted = True
 240   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 241   # connection. If a parent node (e.g. usb hub) is restarted before the
 242   # devices connected to it, the (bus, dev) for the hub can change, making the
 243   # output we have wrong. This way we restart the devices before the hub.
 244   for (bus, dev) in reversed(sorted(usb_devices)):
 245     # Can not restart root usb connections
 246     if dev != '001':
 247       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 248       if return_code:
 249         print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
 250         all_restarted = False
 251       else:
 252         print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
 253
 254   return all_restarted
 255
 256
 257 def KillAllAdb():
 258   def GetAllAdb():
 259     for p in psutil.process_iter():
 260       try:
 261         if 'adb' in p.name:
 262           yield p
 263       except (psutil.NoSuchProcess, psutil.AccessDenied):
 264         pass
 265
 266   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 267     for p in GetAllAdb():
 268       try:
 269         print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
 270             ' '.join(p.cmdline))
 271         p.send_signal(sig)
 272       except (psutil.NoSuchProcess, psutil.AccessDenied):
 273         pass
 274   for p in GetAllAdb():
 275     try:
 276       print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
 277     except (psutil.NoSuchProcess, psutil.AccessDenied):
 278       pass
 279
 280
 281 def main():
 282   parser = optparse.OptionParser()
 283   parser.add_option('', '--out-dir',
 284                     help='Directory where the device path is stored',
 285                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 286   parser.add_option('--no-provisioning-check', action='store_true',
 287                     help='Will not check if devices are provisioned properly.')
 288   parser.add_option('--device-status-dashboard', action='store_true',
 289                     help='Output device status data for dashboard.')
 290   parser.add_option('--restart-usb', action='store_true',
 291                     help='Restart USB ports before running device check.')
 292   parser.add_option('--json-output',
 293                     help='Output JSON information into a specified file.')
 294
 295   options, args = parser.parse_args()
 296   if args:
 297     parser.error('Unknown options %s' % args)
 298
 299   # Remove the last build's "bad devices" before checking device statuses.
 300   device_blacklist.ResetBlacklist()
 301
 302   try:
 303     expected_devices = device_list.GetPersistentDeviceList(
 304         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 305   except IOError:
 306     expected_devices = []
 307   devices = android_commands.GetAttachedDevices()
 308   # Only restart usb if devices are missing.
 309   if set(expected_devices) != set(devices):
 310     print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
 311     KillAllAdb()
 312     retries = 5
 313     usb_restarted = True
 314     if options.restart_usb:
 315       if not RestartUsb():
 316         usb_restarted = False
 317         bb_annotations.PrintWarning()
 318         print 'USB reset stage failed, wait for any device to come back.'
 319     while retries:
 320       print 'retry adb devices...'
 321       time.sleep(1)
 322       devices = android_commands.GetAttachedDevices()
 323       if set(expected_devices) == set(devices):
 324         # All devices are online, keep going.
 325         break
 326       if not usb_restarted and devices:
 327         # The USB wasn't restarted, but there's at least one device online.
 328         # No point in trying to wait for all devices.
 329         break
 330       retries -= 1
 331
 332   # TODO(navabi): Test to make sure this fails and then fix call
 333   offline_devices = android_commands.GetAttachedDevices(
 334       hardware=False, emulator=False, offline=True)
 335
 336   types, builds, batteries, reports, errors, json_data = [], [], [], [], [], []
 337   fail_step_lst = []
 338   if devices:
 339     types, builds, batteries, reports, errors, fail_step_lst, json_data = (
 340         zip(*[DeviceInfo(dev, options) for dev in devices]))
 341
 342   # Write device info to file for buildbot info display.
 343   if os.path.exists('/home/chrome-bot'):
 344     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 345       for device in json_data:
 346         try:
 347           f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
 348               device['build'], float(device['battery']['temperature']) / 10,
 349               device['battery']['level']))
 350         except Exception:
 351           pass
 352
 353   err_msg = CheckForMissingDevices(options, devices) or []
 354
 355   unique_types = list(set(types))
 356   unique_builds = list(set(builds))
 357
 358   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 359                            % (len(devices), unique_types, unique_builds))
 360   print '\n'.join(reports)
 361
 362   for serial, dev_errors in zip(devices, errors):
 363     if dev_errors:
 364       err_msg += ['%s errors:' % serial]
 365       err_msg += ['    %s' % error for error in dev_errors]
 366
 367   if err_msg:
 368     bb_annotations.PrintWarning()
 369     msg = '\n'.join(err_msg)
 370     print msg
 371     from_address = 'buildbot@chromium.org'
 372     to_addresses = ['chromium-android-device-alerts@google.com']
 373     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 374     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 375     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 376     SendEmail(from_address, to_addresses, [], subject, msg)
 377
 378   if options.device_status_dashboard:
 379     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 380                                               [len(devices)], 'devices')
 381     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 382                                               [len(offline_devices)], 'devices',
 383                                               'unimportant')
 384     for serial, battery in zip(devices, batteries):
 385       perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
 386                                                 [battery], '%',
 387                                                 'unimportant')
 388
 389   if options.json_output:
 390     with open(options.json_output, 'wb') as f:
 391       f.write(json.dumps(json_data, indent=4))
 392
 393   num_failed_devs = 0
 394   for fail_status, device in zip(fail_step_lst, devices):
 395     if not fail_status:
 396       device_blacklist.ExtendBlacklist([str(device)])
 397       num_failed_devs += 1
 398
 399   if num_failed_devs == len(devices):
 400     return 2
 401
 402   if not devices:
 403     return 1
 404
 405
 406 if __name__ == '__main__':
 407   sys.exit(main())