build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import android_commands
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import device_blacklist
  34 from pylib.device import device_list
  35 from pylib.device import device_utils
  36
  37 def DeviceInfo(serial, options):
  38   """Gathers info on a device via various adb calls.
  39
  40   Args:
  41     serial: The serial of the attached device to construct info about.
  42
  43   Returns:
  44     Tuple of device type, build id, report as a string, error messages, and
  45     boolean indicating whether or not device can be used for testing.
  46   """
  47
  48   device_adb = device_utils.DeviceUtils(serial)
  49   device_type = device_adb.build_product
  50   device_build = device_adb.build_id
  51   device_build_type = device_adb.build_type
  52   device_product_name = device_adb.product_name
  53
  54   try:
  55     battery_info = device_adb.old_interface.GetBatteryInfo()
  56   except Exception as e:
  57     battery_info = {}
  58     logging.error('Unable to obtain battery info for %s, %s', serial, e)
  59
  60   def _GetData(re_expression, line, lambda_function=lambda x: x):
  61     if not line:
  62       return 'Unknown'
  63     found = re.findall(re_expression, line)
  64     if found and len(found):
  65       return lambda_function(found[0])
  66     return 'Unknown'
  67
  68   battery_level = int(battery_info.get('level', 100))
  69   imei_slice = _GetData(r'Device ID = (\d+)',
  70                         device_adb.old_interface.GetSubscriberInfo(),
  71                         lambda x: x[-6:])
  72   json_data = {
  73     'serial': serial,
  74     'type': device_type,
  75     'build': device_build,
  76     'build_detail': device_adb.GetProp('ro.build.fingerprint'),
  77     'battery': battery_info,
  78     'imei_slice': imei_slice,
  79     'wifi_ip': device_adb.GetProp('dhcp.wlan0.ipaddress'),
  80   }
  81   report = ['Device %s (%s)' % (serial, device_type),
  82             '  Build: %s (%s)' %
  83               (device_build, json_data['build_detail']),
  84             '  Current Battery Service state: ',
  85             '\n'.join(['    %s: %s' % (k, v)
  86                        for k, v in battery_info.iteritems()]),
  87             '  IMEI slice: %s' % imei_slice,
  88             '  Wifi IP: %s' % json_data['wifi_ip'],
  89             '']
  90
  91   errors = []
  92   dev_good = True
  93   if battery_level < 15:
  94     errors += ['Device critically low in battery. Will add to blacklist.']
  95     dev_good = False
  96     if not device_adb.old_interface.IsDeviceCharging():
  97       if device_adb.old_interface.CanControlUsbCharging():
  98         device_adb.old_interface.EnableUsbCharging()
  99       else:
 100         logging.error('Device %s is not charging' % serial)
 101   if not options.no_provisioning_check:
 102     setup_wizard_disabled = (
 103         device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED')
 104     if not setup_wizard_disabled and device_build_type != 'user':
 105       errors += ['Setup wizard not disabled. Was it provisioned correctly?']
 106   if (device_product_name == 'mantaray' and
 107       battery_info.get('AC powered', None) != 'true'):
 108     errors += ['Mantaray device not connected to AC power.']
 109
 110   full_report = '\n'.join(report)
 111
 112   return (device_type, device_build, battery_level, full_report, errors,
 113     dev_good, json_data)
 114
 115
 116 def CheckForMissingDevices(options, adb_online_devs):
 117   """Uses file of previous online devices to detect broken phones.
 118
 119   Args:
 120     options: out_dir parameter of options argument is used as the base
 121              directory to load and update the cache file.
 122     adb_online_devs: A list of serial numbers of the currently visible
 123                      and online attached devices.
 124   """
 125   # TODO(navabi): remove this once the bug that causes different number
 126   # of devices to be detected between calls is fixed.
 127   logger = logging.getLogger()
 128   logger.setLevel(logging.INFO)
 129
 130   out_dir = os.path.abspath(options.out_dir)
 131
 132   # last_devices denotes all known devices prior to this run
 133   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 134   last_missing_devices_path = os.path.join(out_dir,
 135       device_list.LAST_MISSING_DEVICES_FILENAME)
 136   try:
 137     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 138   except IOError:
 139     # Ignore error, file might not exist
 140     last_devices = []
 141
 142   try:
 143     last_missing_devices = device_list.GetPersistentDeviceList(
 144         last_missing_devices_path)
 145   except IOError:
 146     last_missing_devices = []
 147
 148   missing_devs = list(set(last_devices) - set(adb_online_devs))
 149   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 150
 151   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 152     logging.info('new_missing_devs %s' % new_missing_devs)
 153     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 154     bb_annotations.PrintSummaryText(devices_missing_msg)
 155
 156     from_address = 'chrome-bot@chromium.org'
 157     to_addresses = ['chrome-labs-tech-ticket@google.com',
 158                     'chrome-android-device-alert@google.com']
 159     cc_addresses = ['chrome-android-device-alert@google.com']
 160     subject = 'Devices offline on %s, %s, %s' % (
 161       os.environ.get('BUILDBOT_SLAVENAME'),
 162       os.environ.get('BUILDBOT_BUILDERNAME'),
 163       os.environ.get('BUILDBOT_BUILDNUMBER'))
 164     msg = ('Please reboot the following devices:\n%s' %
 165            '\n'.join(map(str, new_missing_devs)))
 166     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 167
 168   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 169   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 170   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 171
 172   if not all_known_devices:
 173     # This can happen if for some reason the .last_devices file is not
 174     # present or if it was empty.
 175     return ['No online devices. Have any devices been plugged in?']
 176   if missing_devs:
 177     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 178     bb_annotations.PrintSummaryText(devices_missing_msg)
 179
 180     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 181     # GetAttachedDevices to compare results.
 182     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 183                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 184                   (urllib.quote('Device Offline'),
 185                    urllib.quote('Buildbot: %s %s\n'
 186                                 'Build: %s\n'
 187                                 '(please don\'t change any labels)' %
 188                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 189                                  os.environ.get('BUILDBOT_SLAVENAME'),
 190                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 191     return ['Current online devices: %s' % adb_online_devs,
 192             '%s are no longer visible. Were they removed?\n' % missing_devs,
 193             'SHERIFF:\n',
 194             '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
 195             'Cache file: %s\n\n' % last_devices_path,
 196             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 197             'adb devices(GetAttachedDevices): %s' % adb_online_devs]
 198   else:
 199     new_devs = set(adb_online_devs) - set(last_devices)
 200     if new_devs and os.path.exists(last_devices_path):
 201       bb_annotations.PrintWarning()
 202       bb_annotations.PrintSummaryText(
 203           '%d new devices detected' % len(new_devs))
 204       print ('New devices detected %s. And now back to your '
 205              'regularly scheduled program.' % list(new_devs))
 206
 207
 208 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 209   msg_body = '\r\n'.join(['From: %s' % from_address,
 210                           'To: %s' % ', '.join(to_addresses),
 211                           'CC: %s' % ', '.join(cc_addresses),
 212                           'Subject: %s' % subject, '', msg])
 213   try:
 214     server = smtplib.SMTP('localhost')
 215     server.sendmail(from_address, to_addresses, msg_body)
 216     server.quit()
 217   except Exception as e:
 218     print 'Failed to send alert email. Error: %s' % e
 219
 220
 221 def RestartUsb():
 222   if not os.path.isfile('/usr/bin/restart_usb'):
 223     print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
 224            'on host (see BUG=305769).')
 225     return False
 226
 227   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 228   lsusb_output, _ = lsusb_proc.communicate()
 229   if lsusb_proc.returncode:
 230     print 'Error: Could not get list of USB ports (i.e. lsusb).'
 231     return lsusb_proc.returncode
 232
 233   usb_devices = [re.findall(r'Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 234                  for lsusb_line in lsusb_output.strip().split('\n')]
 235
 236   all_restarted = True
 237   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 238   # connection. If a parent node (e.g. usb hub) is restarted before the
 239   # devices connected to it, the (bus, dev) for the hub can change, making the
 240   # output we have wrong. This way we restart the devices before the hub.
 241   for (bus, dev) in reversed(sorted(usb_devices)):
 242     # Can not restart root usb connections
 243     if dev != '001':
 244       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 245       if return_code:
 246         print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
 247         all_restarted = False
 248       else:
 249         print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
 250
 251   return all_restarted
 252
 253
 254 def KillAllAdb():
 255   def GetAllAdb():
 256     for p in psutil.process_iter():
 257       try:
 258         if 'adb' in p.name:
 259           yield p
 260       except (psutil.NoSuchProcess, psutil.AccessDenied):
 261         pass
 262
 263   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 264     for p in GetAllAdb():
 265       try:
 266         print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
 267             ' '.join(p.cmdline))
 268         p.send_signal(sig)
 269       except (psutil.NoSuchProcess, psutil.AccessDenied):
 270         pass
 271   for p in GetAllAdb():
 272     try:
 273       print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
 274     except (psutil.NoSuchProcess, psutil.AccessDenied):
 275       pass
 276
 277
 278 def main():
 279   parser = optparse.OptionParser()
 280   parser.add_option('', '--out-dir',
 281                     help='Directory where the device path is stored',
 282                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 283   parser.add_option('--no-provisioning-check', action='store_true',
 284                     help='Will not check if devices are provisioned properly.')
 285   parser.add_option('--device-status-dashboard', action='store_true',
 286                     help='Output device status data for dashboard.')
 287   parser.add_option('--restart-usb', action='store_true',
 288                     help='Restart USB ports before running device check.')
 289   parser.add_option('--json-output',
 290                     help='Output JSON information into a specified file.')
 291
 292   options, args = parser.parse_args()
 293   if args:
 294     parser.error('Unknown options %s' % args)
 295
 296   # Remove the last build's "bad devices" before checking device statuses.
 297   device_blacklist.ResetBlacklist()
 298
 299   try:
 300     expected_devices = device_list.GetPersistentDeviceList(
 301         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 302   except IOError:
 303     expected_devices = []
 304   devices = android_commands.GetAttachedDevices()
 305   # Only restart usb if devices are missing.
 306   if set(expected_devices) != set(devices):
 307     print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
 308     KillAllAdb()
 309     retries = 5
 310     usb_restarted = True
 311     if options.restart_usb:
 312       if not RestartUsb():
 313         usb_restarted = False
 314         bb_annotations.PrintWarning()
 315         print 'USB reset stage failed, wait for any device to come back.'
 316     while retries:
 317       print 'retry adb devices...'
 318       time.sleep(1)
 319       devices = android_commands.GetAttachedDevices()
 320       if set(expected_devices) == set(devices):
 321         # All devices are online, keep going.
 322         break
 323       if not usb_restarted and devices:
 324         # The USB wasn't restarted, but there's at least one device online.
 325         # No point in trying to wait for all devices.
 326         break
 327       retries -= 1
 328
 329   # TODO(navabi): Test to make sure this fails and then fix call
 330   offline_devices = android_commands.GetAttachedDevices(
 331       hardware=False, emulator=False, offline=True)
 332
 333   types, builds, batteries, reports, errors, json_data = [], [], [], [], [], []
 334   fail_step_lst = []
 335   if devices:
 336     types, builds, batteries, reports, errors, fail_step_lst, json_data = (
 337         zip(*[DeviceInfo(dev, options) for dev in devices]))
 338
 339   # Write device info to file for buildbot info display.
 340   if os.path.exists('/home/chrome-bot'):
 341     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 342       for device in json_data:
 343         try:
 344           f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
 345               device['build'], float(device['battery']['temperature']) / 10,
 346               device['battery']['level']))
 347         except Exception:
 348           pass
 349
 350   err_msg = CheckForMissingDevices(options, devices) or []
 351
 352   unique_types = list(set(types))
 353   unique_builds = list(set(builds))
 354
 355   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 356                            % (len(devices), unique_types, unique_builds))
 357   print '\n'.join(reports)
 358
 359   for serial, dev_errors in zip(devices, errors):
 360     if dev_errors:
 361       err_msg += ['%s errors:' % serial]
 362       err_msg += ['    %s' % error for error in dev_errors]
 363
 364   if err_msg:
 365     bb_annotations.PrintWarning()
 366     msg = '\n'.join(err_msg)
 367     print msg
 368     from_address = 'buildbot@chromium.org'
 369     to_addresses = ['chromium-android-device-alerts@google.com']
 370     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 371     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 372     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 373     SendEmail(from_address, to_addresses, [], subject, msg)
 374
 375   if options.device_status_dashboard:
 376     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 377                                               [len(devices)], 'devices')
 378     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 379                                               [len(offline_devices)], 'devices',
 380                                               'unimportant')
 381     for serial, battery in zip(devices, batteries):
 382       perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
 383                                                 [battery], '%',
 384                                                 'unimportant')
 385
 386   if options.json_output:
 387     with open(options.json_output, 'wb') as f:
 388       f.write(json.dumps(json_data, indent=4))
 389
 390   num_failed_devs = 0
 391   for fail_status, device in zip(fail_step_lst, devices):
 392     if not fail_status:
 393       device_blacklist.ExtendBlacklist([str(device)])
 394       num_failed_devs += 1
 395
 396   if num_failed_devs == len(devices):
 397     return 2
 398
 399   if not devices:
 400     return 1
 401
 402
 403 if __name__ == '__main__':
 404   sys.exit(main())