build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import android_commands
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import device_blacklist
  34 from pylib.device import device_errors
  35 from pylib.device import device_list
  36 from pylib.device import device_utils
  37
  38 def DeviceInfo(serial, options):
  39   """Gathers info on a device via various adb calls.
  40
  41   Args:
  42     serial: The serial of the attached device to construct info about.
  43
  44   Returns:
  45     Tuple of device type, build id, report as a string, error messages, and
  46     boolean indicating whether or not device can be used for testing.
  47   """
  48
  49   device_adb = device_utils.DeviceUtils(serial)
  50   device_type = device_adb.GetProp('ro.build.product')
  51   device_build = device_adb.GetProp('ro.build.id')
  52   device_build_type = device_adb.GetProp('ro.build.type')
  53   device_product_name = device_adb.GetProp('ro.product.name')
  54
  55   try:
  56     battery_info = device_adb.old_interface.GetBatteryInfo()
  57   except Exception as e:
  58     battery_info = {}
  59     logging.error('Unable to obtain battery info for %s, %s', serial, e)
  60
  61   def _GetData(re_expression, line, lambda_function=lambda x:x):
  62     if not line:
  63       return 'Unknown'
  64     found = re.findall(re_expression, line)
  65     if found and len(found):
  66       return lambda_function(found[0])
  67     return 'Unknown'
  68
  69   battery_level = int(battery_info.get('level', 100))
  70   imei_slice = _GetData('Device ID = (\d+)',
  71                         device_adb.old_interface.GetSubscriberInfo(),
  72                         lambda x: x[-6:])
  73   report = ['Device %s (%s)' % (serial, device_type),
  74             '  Build: %s (%s)' %
  75               (device_build, device_adb.GetProp('ro.build.fingerprint')),
  76             '  Current Battery Service state: ',
  77             '\n'.join(['    %s: %s' % (k, v)
  78                        for k, v in battery_info.iteritems()]),
  79             '  IMEI slice: %s' % imei_slice,
  80             '  Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'),
  81             '']
  82
  83   errors = []
  84   dev_good = True
  85   if battery_level < 15:
  86     errors += ['Device critically low in battery. Turning off device.']
  87     dev_good = False
  88   if not options.no_provisioning_check:
  89     setup_wizard_disabled = (
  90         device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED')
  91     if not setup_wizard_disabled and device_build_type != 'user':
  92       errors += ['Setup wizard not disabled. Was it provisioned correctly?']
  93   if (device_product_name == 'mantaray' and
  94       battery_info.get('AC powered', None) != 'true'):
  95     errors += ['Mantaray device not connected to AC power.']
  96
  97   # Turn off devices with low battery.
  98   if battery_level < 15:
  99     try:
 100       device_adb.EnableRoot()
 101     except device_errors.CommandFailedError as e:
 102       # Attempt shutdown anyway.
 103       # TODO(jbudorick) Handle this exception appropriately after interface
 104       #                 conversions are finished.
 105       logging.error(str(e))
 106     device_adb.old_interface.Shutdown()
 107   full_report = '\n'.join(report)
 108   return device_type, device_build, battery_level, full_report, errors, dev_good
 109
 110
 111 def CheckForMissingDevices(options, adb_online_devs):
 112   """Uses file of previous online devices to detect broken phones.
 113
 114   Args:
 115     options: out_dir parameter of options argument is used as the base
 116              directory to load and update the cache file.
 117     adb_online_devs: A list of serial numbers of the currently visible
 118                      and online attached devices.
 119   """
 120   # TODO(navabi): remove this once the bug that causes different number
 121   # of devices to be detected between calls is fixed.
 122   logger = logging.getLogger()
 123   logger.setLevel(logging.INFO)
 124
 125   out_dir = os.path.abspath(options.out_dir)
 126
 127   # last_devices denotes all known devices prior to this run
 128   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 129   last_missing_devices_path = os.path.join(out_dir,
 130       device_list.LAST_MISSING_DEVICES_FILENAME)
 131   try:
 132     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 133   except IOError:
 134     # Ignore error, file might not exist
 135     last_devices = []
 136
 137   try:
 138     last_missing_devices = device_list.GetPersistentDeviceList(
 139         last_missing_devices_path)
 140   except IOError:
 141     last_missing_devices = []
 142
 143   missing_devs = list(set(last_devices) - set(adb_online_devs))
 144   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 145
 146   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 147     logging.info('new_missing_devs %s' % new_missing_devs)
 148     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 149     bb_annotations.PrintSummaryText(devices_missing_msg)
 150
 151     from_address = 'chrome-bot@chromium.org'
 152     to_addresses = ['chrome-labs-tech-ticket@google.com',
 153                     'chrome-android-device-alert@google.com']
 154     cc_addresses = ['chrome-android-device-alert@google.com']
 155     subject = 'Devices offline on %s, %s, %s' % (
 156       os.environ.get('BUILDBOT_SLAVENAME'),
 157       os.environ.get('BUILDBOT_BUILDERNAME'),
 158       os.environ.get('BUILDBOT_BUILDNUMBER'))
 159     msg = ('Please reboot the following devices:\n%s' %
 160            '\n'.join(map(str,new_missing_devs)))
 161     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 162
 163   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 164   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 165   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 166
 167   if not all_known_devices:
 168     # This can happen if for some reason the .last_devices file is not
 169     # present or if it was empty.
 170     return ['No online devices. Have any devices been plugged in?']
 171   if missing_devs:
 172     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 173     bb_annotations.PrintSummaryText(devices_missing_msg)
 174
 175     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 176     # GetAttachedDevices to compare results.
 177     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 178                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 179                   (urllib.quote('Device Offline'),
 180                    urllib.quote('Buildbot: %s %s\n'
 181                                 'Build: %s\n'
 182                                 '(please don\'t change any labels)' %
 183                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 184                                  os.environ.get('BUILDBOT_SLAVENAME'),
 185                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 186     return ['Current online devices: %s' % adb_online_devs,
 187             '%s are no longer visible. Were they removed?\n' % missing_devs,
 188             'SHERIFF:\n',
 189             '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
 190             'Cache file: %s\n\n' % last_devices_path,
 191             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 192             'adb devices(GetAttachedDevices): %s' % adb_online_devs]
 193   else:
 194     new_devs = set(adb_online_devs) - set(last_devices)
 195     if new_devs and os.path.exists(last_devices_path):
 196       bb_annotations.PrintWarning()
 197       bb_annotations.PrintSummaryText(
 198           '%d new devices detected' % len(new_devs))
 199       print ('New devices detected %s. And now back to your '
 200              'regularly scheduled program.' % list(new_devs))
 201
 202
 203 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 204   msg_body = '\r\n'.join(['From: %s' % from_address,
 205                           'To: %s' % ', '.join(to_addresses),
 206                           'CC: %s' % ', '.join(cc_addresses),
 207                           'Subject: %s' % subject, '', msg])
 208   try:
 209     server = smtplib.SMTP('localhost')
 210     server.sendmail(from_address, to_addresses, msg_body)
 211     server.quit()
 212   except Exception as e:
 213     print 'Failed to send alert email. Error: %s' % e
 214
 215
 216 def RestartUsb():
 217   if not os.path.isfile('/usr/bin/restart_usb'):
 218     print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
 219            'on host (see BUG=305769).')
 220     return False
 221
 222   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 223   lsusb_output, _ = lsusb_proc.communicate()
 224   if lsusb_proc.returncode:
 225     print ('Error: Could not get list of USB ports (i.e. lsusb).')
 226     return lsusb_proc.returncode
 227
 228   usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 229                  for lsusb_line in lsusb_output.strip().split('\n')]
 230
 231   all_restarted = True
 232   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 233   # connection. If a parent node (e.g. usb hub) is restarted before the
 234   # devices connected to it, the (bus, dev) for the hub can change, making the
 235   # output we have wrong. This way we restart the devices before the hub.
 236   for (bus, dev) in reversed(sorted(usb_devices)):
 237     # Can not restart root usb connections
 238     if dev != '001':
 239       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 240       if return_code:
 241         print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
 242         all_restarted = False
 243       else:
 244         print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
 245
 246   return all_restarted
 247
 248
 249 def KillAllAdb():
 250   def GetAllAdb():
 251     for p in psutil.process_iter():
 252       try:
 253         if 'adb' in p.name:
 254           yield p
 255       except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 256         pass
 257
 258   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 259     for p in GetAllAdb():
 260       try:
 261         print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
 262             ' '.join(p.cmdline))
 263         p.send_signal(sig)
 264       except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 265         pass
 266   for p in GetAllAdb():
 267     try:
 268       print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
 269     except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 270       pass
 271
 272
 273 def main():
 274   parser = optparse.OptionParser()
 275   parser.add_option('', '--out-dir',
 276                     help='Directory where the device path is stored',
 277                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 278   parser.add_option('--no-provisioning-check', action='store_true',
 279                     help='Will not check if devices are provisioned properly.')
 280   parser.add_option('--device-status-dashboard', action='store_true',
 281                     help='Output device status data for dashboard.')
 282   parser.add_option('--restart-usb', action='store_true',
 283                     help='Restart USB ports before running device check.')
 284   parser.add_option('--json-output',
 285                     help='Output JSON information into a specified file.')
 286
 287   options, args = parser.parse_args()
 288   if args:
 289     parser.error('Unknown options %s' % args)
 290
 291   # Remove the last build's "bad devices" before checking device statuses.
 292   device_blacklist.ResetBlacklist()
 293
 294   try:
 295     expected_devices = device_list.GetPersistentDeviceList(
 296         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 297   except IOError:
 298     expected_devices = []
 299   devices = android_commands.GetAttachedDevices()
 300   # Only restart usb if devices are missing.
 301   if set(expected_devices) != set(devices):
 302     print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
 303     KillAllAdb()
 304     retries = 5
 305     usb_restarted = True
 306     if options.restart_usb:
 307       if not RestartUsb():
 308         usb_restarted = False
 309         bb_annotations.PrintWarning()
 310         print 'USB reset stage failed, wait for any device to come back.'
 311     while retries:
 312       print 'retry adb devices...'
 313       time.sleep(1)
 314       devices = android_commands.GetAttachedDevices()
 315       if set(expected_devices) == set(devices):
 316         # All devices are online, keep going.
 317         break
 318       if not usb_restarted and devices:
 319         # The USB wasn't restarted, but there's at least one device online.
 320         # No point in trying to wait for all devices.
 321         break
 322       retries -= 1
 323
 324   # TODO(navabi): Test to make sure this fails and then fix call
 325   offline_devices = android_commands.GetAttachedDevices(
 326       hardware=False, emulator=False, offline=True)
 327
 328   types, builds, batteries, reports, errors = [], [], [], [], []
 329   fail_step_lst = []
 330   if devices:
 331     types, builds, batteries, reports, errors, fail_step_lst = (
 332         zip(*[DeviceInfo(dev, options) for dev in devices]))
 333
 334   err_msg = CheckForMissingDevices(options, devices) or []
 335
 336   unique_types = list(set(types))
 337   unique_builds = list(set(builds))
 338
 339   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 340                            % (len(devices), unique_types, unique_builds))
 341   print '\n'.join(reports)
 342
 343   for serial, dev_errors in zip(devices, errors):
 344     if dev_errors:
 345       err_msg += ['%s errors:' % serial]
 346       err_msg += ['    %s' % error for error in dev_errors]
 347
 348   if err_msg:
 349     bb_annotations.PrintWarning()
 350     msg = '\n'.join(err_msg)
 351     print msg
 352     from_address = 'buildbot@chromium.org'
 353     to_addresses = ['chromium-android-device-alerts@google.com']
 354     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 355     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 356     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 357     SendEmail(from_address, to_addresses, [], subject, msg)
 358
 359   if options.device_status_dashboard:
 360     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 361                                               [len(devices)], 'devices')
 362     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 363                                               [len(offline_devices)], 'devices',
 364                                               'unimportant')
 365     for serial, battery in zip(devices, batteries):
 366       perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
 367                                                 [battery], '%',
 368                                                 'unimportant')
 369
 370   if options.json_output:
 371     with open(options.json_output, 'wb') as f:
 372       f.write(json.dumps({
 373         'online_devices': devices,
 374         'offline_devices': offline_devices,
 375         'expected_devices': expected_devices,
 376         'unique_types': unique_types,
 377         'unique_builds': unique_builds,
 378       }))
 379
 380   if False in fail_step_lst:
 381     # TODO(navabi): Build fails on device status check step if there exists any
 382     # devices with critically low battery. Remove those devices from testing,
 383     # allowing build to continue with good devices.
 384     return 2
 385
 386   if not devices:
 387     return 1
 388
 389
 390 if __name__ == '__main__':
 391   sys.exit(main())