build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import android_commands
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import device_blacklist
  34 from pylib.device import device_errors
  35 from pylib.device import device_list
  36 from pylib.device import device_utils
  37
  38 def DeviceInfo(serial, options):
  39   """Gathers info on a device via various adb calls.
  40
  41   Args:
  42     serial: The serial of the attached device to construct info about.
  43
  44   Returns:
  45     Tuple of device type, build id, report as a string, error messages, and
  46     boolean indicating whether or not device can be used for testing.
  47   """
  48
  49   device_adb = device_utils.DeviceUtils(serial)
  50   device_type = device_adb.GetProp('ro.build.product')
  51   device_build = device_adb.GetProp('ro.build.id')
  52   device_build_type = device_adb.GetProp('ro.build.type')
  53   device_product_name = device_adb.GetProp('ro.product.name')
  54
  55   try:
  56     battery_info = device_adb.old_interface.GetBatteryInfo()
  57   except Exception as e:
  58     battery_info = {}
  59     logging.error('Unable to obtain battery info for %s, %s', serial, e)
  60
  61   def _GetData(re_expression, line, lambda_function=lambda x:x):
  62     if not line:
  63       return 'Unknown'
  64     found = re.findall(re_expression, line)
  65     if found and len(found):
  66       return lambda_function(found[0])
  67     return 'Unknown'
  68
  69   battery_level = int(battery_info.get('level', 100))
  70   imei_slice = _GetData('Device ID = (\d+)',
  71                         device_adb.old_interface.GetSubscriberInfo(),
  72                         lambda x: x[-6:])
  73   report = ['Device %s (%s)' % (serial, device_type),
  74             '  Build: %s (%s)' %
  75               (device_build, device_adb.GetProp('ro.build.fingerprint')),
  76             '  Current Battery Service state: ',
  77             '\n'.join(['    %s: %s' % (k, v)
  78                        for k, v in battery_info.iteritems()]),
  79             '  IMEI slice: %s' % imei_slice,
  80             '  Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'),
  81             '']
  82
  83   errors = []
  84   dev_good = True
  85   if battery_level < 15:
  86     errors += ['Device critically low in battery. Turning off device.']
  87     dev_good = False
  88   if not options.no_provisioning_check:
  89     setup_wizard_disabled = (
  90         device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED')
  91     if not setup_wizard_disabled and device_build_type != 'user':
  92       errors += ['Setup wizard not disabled. Was it provisioned correctly?']
  93   if (device_product_name == 'mantaray' and
  94       battery_info.get('AC powered', None) != 'true'):
  95     errors += ['Mantaray device not connected to AC power.']
  96
  97   # Turn off devices with low battery.
  98   if battery_level < 15:
  99     try:
 100       device_adb.EnableRoot()
 101     except device_errors.CommandFailedError as e:
 102       # Attempt shutdown anyway.
 103       # TODO(jbudorick) Handle this exception appropriately after interface
 104       #                 conversions are finished.
 105       logging.error(str(e))
 106     device_adb.old_interface.Shutdown()
 107   full_report = '\n'.join(report)
 108   return device_type, device_build, battery_level, full_report, errors, dev_good
 109
 110
 111 def CheckForMissingDevices(options, adb_online_devs):
 112   """Uses file of previous online devices to detect broken phones.
 113
 114   Args:
 115     options: out_dir parameter of options argument is used as the base
 116              directory to load and update the cache file.
 117     adb_online_devs: A list of serial numbers of the currently visible
 118                      and online attached devices.
 119   """
 120   # TODO(navabi): remove this once the bug that causes different number
 121   # of devices to be detected between calls is fixed.
 122   logger = logging.getLogger()
 123   logger.setLevel(logging.INFO)
 124
 125   out_dir = os.path.abspath(options.out_dir)
 126
 127   # last_devices denotes all known devices prior to this run
 128   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 129   last_missing_devices_path = os.path.join(out_dir,
 130       device_list.LAST_MISSING_DEVICES_FILENAME)
 131   try:
 132     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 133   except IOError:
 134     # Ignore error, file might not exist
 135     last_devices = []
 136
 137   try:
 138     last_missing_devices = device_list.GetPersistentDeviceList(
 139         last_missing_devices_path)
 140   except IOError:
 141     last_missing_devices = []
 142
 143   missing_devs = list(set(last_devices) - set(adb_online_devs))
 144   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 145
 146   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 147     logging.info('new_missing_devs %s' % new_missing_devs)
 148     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 149     bb_annotations.PrintSummaryText(devices_missing_msg)
 150
 151     from_address = 'chrome-bot@chromium.org'
 152     to_addresses = ['chrome-labs-tech-ticket@google.com']
 153     subject = 'Devices offline on %s, %s, %s' % (
 154       os.environ.get('BUILDBOT_SLAVENAME'),
 155       os.environ.get('BUILDBOT_BUILDERNAME'),
 156       os.environ.get('BUILDBOT_BUILDNUMBER'))
 157     msg = ('Please reboot the following devices:\n%s' %
 158            '\n'.join(map(str,new_missing_devs)))
 159     SendEmail(from_address, to_addresses, subject, msg)
 160
 161   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 162   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 163   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 164
 165   if not all_known_devices:
 166     # This can happen if for some reason the .last_devices file is not
 167     # present or if it was empty.
 168     return ['No online devices. Have any devices been plugged in?']
 169   if missing_devs:
 170     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 171     bb_annotations.PrintSummaryText(devices_missing_msg)
 172
 173     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 174     # GetAttachedDevices to compare results.
 175     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 176                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 177                   (urllib.quote('Device Offline'),
 178                    urllib.quote('Buildbot: %s %s\n'
 179                                 'Build: %s\n'
 180                                 '(please don\'t change any labels)' %
 181                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 182                                  os.environ.get('BUILDBOT_SLAVENAME'),
 183                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 184     return ['Current online devices: %s' % adb_online_devs,
 185             '%s are no longer visible. Were they removed?\n' % missing_devs,
 186             'SHERIFF:\n',
 187             '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
 188             'Cache file: %s\n\n' % last_devices_path,
 189             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 190             'adb devices(GetAttachedDevices): %s' % adb_online_devs]
 191   else:
 192     new_devs = set(adb_online_devs) - set(last_devices)
 193     if new_devs and os.path.exists(last_devices_path):
 194       bb_annotations.PrintWarning()
 195       bb_annotations.PrintSummaryText(
 196           '%d new devices detected' % len(new_devs))
 197       print ('New devices detected %s. And now back to your '
 198              'regularly scheduled program.' % list(new_devs))
 199
 200
 201 def SendEmail(from_address, to_addresses, subject, msg):
 202   msg_body = '\r\n'.join(['From: %s' % from_address,
 203                           'To: %s' % ', '.join(to_addresses),
 204                           'Subject: %s' % subject, '', msg])
 205   try:
 206     server = smtplib.SMTP('localhost')
 207     server.sendmail(from_address, to_addresses, msg_body)
 208     server.quit()
 209   except Exception as e:
 210     print 'Failed to send alert email. Error: %s' % e
 211
 212
 213 def RestartUsb():
 214   if not os.path.isfile('/usr/bin/restart_usb'):
 215     print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
 216            'on host (see BUG=305769).')
 217     return False
 218
 219   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 220   lsusb_output, _ = lsusb_proc.communicate()
 221   if lsusb_proc.returncode:
 222     print ('Error: Could not get list of USB ports (i.e. lsusb).')
 223     return lsusb_proc.returncode
 224
 225   usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 226                  for lsusb_line in lsusb_output.strip().split('\n')]
 227
 228   all_restarted = True
 229   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 230   # connection. If a parent node (e.g. usb hub) is restarted before the
 231   # devices connected to it, the (bus, dev) for the hub can change, making the
 232   # output we have wrong. This way we restart the devices before the hub.
 233   for (bus, dev) in reversed(sorted(usb_devices)):
 234     # Can not restart root usb connections
 235     if dev != '001':
 236       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 237       if return_code:
 238         print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
 239         all_restarted = False
 240       else:
 241         print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
 242
 243   return all_restarted
 244
 245
 246 def KillAllAdb():
 247   def GetAllAdb():
 248     for p in psutil.process_iter():
 249       try:
 250         if 'adb' in p.name:
 251           yield p
 252       except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 253         pass
 254
 255   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 256     for p in GetAllAdb():
 257       try:
 258         print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
 259             ' '.join(p.cmdline))
 260         p.send_signal(sig)
 261       except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 262         pass
 263   for p in GetAllAdb():
 264     try:
 265       print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
 266     except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
 267       pass
 268
 269
 270 def main():
 271   parser = optparse.OptionParser()
 272   parser.add_option('', '--out-dir',
 273                     help='Directory where the device path is stored',
 274                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 275   parser.add_option('--no-provisioning-check', action='store_true',
 276                     help='Will not check if devices are provisioned properly.')
 277   parser.add_option('--device-status-dashboard', action='store_true',
 278                     help='Output device status data for dashboard.')
 279   parser.add_option('--restart-usb', action='store_true',
 280                     help='Restart USB ports before running device check.')
 281   parser.add_option('--json-output',
 282                     help='Output JSON information into a specified file.')
 283
 284   options, args = parser.parse_args()
 285   if args:
 286     parser.error('Unknown options %s' % args)
 287
 288   # Remove the last build's "bad devices" before checking device statuses.
 289   device_blacklist.ResetBlacklist()
 290
 291   try:
 292     expected_devices = device_list.GetPersistentDeviceList(
 293         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 294   except IOError:
 295     expected_devices = []
 296   devices = android_commands.GetAttachedDevices()
 297   # Only restart usb if devices are missing.
 298   if set(expected_devices) != set(devices):
 299     print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
 300     KillAllAdb()
 301     retries = 5
 302     usb_restarted = True
 303     if options.restart_usb:
 304       if not RestartUsb():
 305         usb_restarted = False
 306         bb_annotations.PrintWarning()
 307         print 'USB reset stage failed, wait for any device to come back.'
 308     while retries:
 309       print 'retry adb devices...'
 310       time.sleep(1)
 311       devices = android_commands.GetAttachedDevices()
 312       if set(expected_devices) == set(devices):
 313         # All devices are online, keep going.
 314         break
 315       if not usb_restarted and devices:
 316         # The USB wasn't restarted, but there's at least one device online.
 317         # No point in trying to wait for all devices.
 318         break
 319       retries -= 1
 320
 321   # TODO(navabi): Test to make sure this fails and then fix call
 322   offline_devices = android_commands.GetAttachedDevices(
 323       hardware=False, emulator=False, offline=True)
 324
 325   types, builds, batteries, reports, errors = [], [], [], [], []
 326   fail_step_lst = []
 327   if devices:
 328     types, builds, batteries, reports, errors, fail_step_lst = (
 329         zip(*[DeviceInfo(dev, options) for dev in devices]))
 330
 331   err_msg = CheckForMissingDevices(options, devices) or []
 332
 333   unique_types = list(set(types))
 334   unique_builds = list(set(builds))
 335
 336   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 337                            % (len(devices), unique_types, unique_builds))
 338   print '\n'.join(reports)
 339
 340   for serial, dev_errors in zip(devices, errors):
 341     if dev_errors:
 342       err_msg += ['%s errors:' % serial]
 343       err_msg += ['    %s' % error for error in dev_errors]
 344
 345   if err_msg:
 346     bb_annotations.PrintWarning()
 347     msg = '\n'.join(err_msg)
 348     print msg
 349     from_address = 'buildbot@chromium.org'
 350     to_addresses = ['chromium-android-device-alerts@google.com']
 351     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 352     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 353     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 354     SendEmail(from_address, to_addresses, subject, msg)
 355
 356   if options.device_status_dashboard:
 357     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 358                                               [len(devices)], 'devices')
 359     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 360                                               [len(offline_devices)], 'devices',
 361                                               'unimportant')
 362     for serial, battery in zip(devices, batteries):
 363       perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
 364                                                 [battery], '%',
 365                                                 'unimportant')
 366
 367   if options.json_output:
 368     with open(options.json_output, 'wb') as f:
 369       f.write(json.dumps({
 370         'online_devices': devices,
 371         'offline_devices': offline_devices,
 372         'expected_devices': expected_devices,
 373         'unique_types': unique_types,
 374         'unique_builds': unique_builds,
 375       }))
 376
 377   if False in fail_step_lst:
 378     # TODO(navabi): Build fails on device status check step if there exists any
 379     # devices with critically low battery. Remove those devices from testing,
 380     # allowing build to continue with good devices.
 381     return 2
 382
 383   if not devices:
 384     return 1
 385
 386
 387 if __name__ == '__main__':
 388   sys.exit(main())