build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import json
   9 import logging
  10 import optparse
  11 import os
  12 import psutil
  13 import re
  14 import signal
  15 import smtplib
  16 import subprocess
  17 import sys
  18 import time
  19 import urllib
  20
  21 import bb_annotations
  22 import bb_utils
  23
  24 sys.path.append(os.path.join(os.path.dirname(__file__),
  25                              os.pardir, os.pardir, 'util', 'lib',
  26                              'common'))
  27 import perf_tests_results_helper  # pylint: disable=F0401
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  30 from pylib import android_commands
  31 from pylib import constants
  32 from pylib.cmd_helper import GetCmdOutput
  33 from pylib.device import battery_utils
  34 from pylib.device import device_blacklist
  35 from pylib.device import device_errors
  36 from pylib.device import device_list
  37 from pylib.device import device_utils
  38 from pylib.utils import run_tests_helper
  39
  40 _RE_DEVICE_ID = re.compile('Device ID = (\d+)')
  41
  42 def DeviceInfo(serial, options):
  43   """Gathers info on a device via various adb calls.
  44
  45   Args:
  46     serial: The serial of the attached device to construct info about.
  47
  48   Returns:
  49     Tuple of device type, build id, report as a string, error messages, and
  50     boolean indicating whether or not device can be used for testing.
  51   """
  52   device = device_utils.DeviceUtils(serial)
  53   battery = battery_utils.BatteryUtils(device)
  54
  55   battery_info = {}
  56   battery_level = 100
  57   errors = []
  58   dev_good = True
  59   json_data = {
  60     'serial': serial,
  61     'type': device.build_product,
  62     'build': device.build_id,
  63     'build_detail': device.GetProp('ro.build.fingerprint'),
  64     'battery': {},
  65     'imei_slice': 'Unknown',
  66     'wifi_ip': device.GetProp('dhcp.wlan0.ipaddress'),
  67   }
  68
  69   try:
  70     try:
  71       battery_info = battery.GetBatteryInfo(timeout=5)
  72       battery_level = int(battery_info.get('level', battery_level))
  73       json_data['battery'] = battery_info
  74     except device_errors.CommandFailedError:
  75       logging.exception('Failed to get battery information for %s', serial)
  76
  77     try:
  78       for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
  79                                       check_return=True, timeout=5):
  80         m = _RE_DEVICE_ID.match(l)
  81         if m:
  82           json_data['imei_slice'] = m.group(1)[-6:]
  83     except device_errors.CommandFailedError:
  84       logging.exception('Failed to get IMEI slice for %s', serial)
  85
  86     if battery_level < 15:
  87       errors += ['Device critically low in battery.']
  88       dev_good = False
  89       if not battery.GetCharging():
  90         battery.SetCharging(True)
  91     if not options.no_provisioning_check:
  92       setup_wizard_disabled = (
  93           device.GetProp('ro.setupwizard.mode') == 'DISABLED')
  94       if not setup_wizard_disabled and device.build_type != 'user':
  95         errors += ['Setup wizard not disabled. Was it provisioned correctly?']
  96     if (device.product_name == 'mantaray' and
  97         battery_info.get('AC powered', None) != 'true'):
  98       errors += ['Mantaray device not connected to AC power.']
  99   except device_errors.CommandFailedError:
 100     logging.exception('Failure while getting device status.')
 101     dev_good = False
 102   except device_errors.CommandTimeoutError:
 103     logging.exception('Timeout while getting device status.')
 104     dev_good = False
 105
 106   return (device.build_product, device.build_id, battery_level, errors,
 107           dev_good, json_data)
 108
 109
 110 def CheckForMissingDevices(options, adb_online_devs):
 111   """Uses file of previous online devices to detect broken phones.
 112
 113   Args:
 114     options: out_dir parameter of options argument is used as the base
 115              directory to load and update the cache file.
 116     adb_online_devs: A list of serial numbers of the currently visible
 117                      and online attached devices.
 118   """
 119   out_dir = os.path.abspath(options.out_dir)
 120
 121   # last_devices denotes all known devices prior to this run
 122   last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
 123   last_missing_devices_path = os.path.join(out_dir,
 124       device_list.LAST_MISSING_DEVICES_FILENAME)
 125   try:
 126     last_devices = device_list.GetPersistentDeviceList(last_devices_path)
 127   except IOError:
 128     # Ignore error, file might not exist
 129     last_devices = []
 130
 131   try:
 132     last_missing_devices = device_list.GetPersistentDeviceList(
 133         last_missing_devices_path)
 134   except IOError:
 135     last_missing_devices = []
 136
 137   missing_devs = list(set(last_devices) - set(adb_online_devs))
 138   new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
 139
 140   if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
 141     logging.info('new_missing_devs %s' % new_missing_devs)
 142     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 143     bb_annotations.PrintSummaryText(devices_missing_msg)
 144
 145     from_address = 'chrome-bot@chromium.org'
 146     to_addresses = ['chrome-labs-tech-ticket@google.com',
 147                     'chrome-android-device-alert@google.com']
 148     cc_addresses = ['chrome-android-device-alert@google.com']
 149     subject = 'Devices offline on %s, %s, %s' % (
 150       os.environ.get('BUILDBOT_SLAVENAME'),
 151       os.environ.get('BUILDBOT_BUILDERNAME'),
 152       os.environ.get('BUILDBOT_BUILDNUMBER'))
 153     msg = ('Please reboot the following devices:\n%s' %
 154            '\n'.join(map(str, new_missing_devs)))
 155     SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
 156
 157   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 158   device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
 159   device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
 160
 161   if not all_known_devices:
 162     # This can happen if for some reason the .last_devices file is not
 163     # present or if it was empty.
 164     return ['No online devices. Have any devices been plugged in?']
 165   if missing_devs:
 166     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 167     bb_annotations.PrintSummaryText(devices_missing_msg)
 168
 169     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 170     # GetAttachedDevices to compare results.
 171     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 172                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 173                   (urllib.quote('Device Offline'),
 174                    urllib.quote('Buildbot: %s %s\n'
 175                                 'Build: %s\n'
 176                                 '(please don\'t change any labels)' %
 177                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 178                                  os.environ.get('BUILDBOT_SLAVENAME'),
 179                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 180     return ['Current online devices: %s' % adb_online_devs,
 181             '%s are no longer visible. Were they removed?' % missing_devs,
 182             'SHERIFF:',
 183             '@@@STEP_LINK@Click here to file a bug@%s@@@' % crbug_link,
 184             'Cache file: %s' % last_devices_path,
 185             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 186             'adb devices(GetAttachedDevices): %s' % adb_online_devs]
 187   else:
 188     new_devs = set(adb_online_devs) - set(last_devices)
 189     if new_devs and os.path.exists(last_devices_path):
 190       bb_annotations.PrintWarning()
 191       bb_annotations.PrintSummaryText(
 192           '%d new devices detected' % len(new_devs))
 193       logging.info('New devices detected:')
 194       for d in new_devs:
 195         logging.info('  %s', d)
 196
 197
 198 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
 199   msg_body = '\r\n'.join(['From: %s' % from_address,
 200                           'To: %s' % ', '.join(to_addresses),
 201                           'CC: %s' % ', '.join(cc_addresses),
 202                           'Subject: %s' % subject, '', msg])
 203   try:
 204     server = smtplib.SMTP('localhost')
 205     server.sendmail(from_address, to_addresses, msg_body)
 206     server.quit()
 207   except Exception:
 208     logging.exception('Failed to send alert email.')
 209
 210
 211 def RestartUsb():
 212   if not os.path.isfile('/usr/bin/restart_usb'):
 213     logging.error('Could not restart usb. ''/usr/bin/restart_usb not '
 214                   'installed on host (see BUG=305769).')
 215     return False
 216
 217   lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
 218   lsusb_output, _ = lsusb_proc.communicate()
 219   if lsusb_proc.returncode:
 220     logging.error('Could not get list of USB ports (i.e. lsusb).')
 221     return lsusb_proc.returncode
 222
 223   usb_devices = [re.findall(r'Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
 224                  for lsusb_line in lsusb_output.strip().split('\n')]
 225
 226   all_restarted = True
 227   # Walk USB devices from leaves up (i.e reverse sorted) restarting the
 228   # connection. If a parent node (e.g. usb hub) is restarted before the
 229   # devices connected to it, the (bus, dev) for the hub can change, making the
 230   # output we have wrong. This way we restart the devices before the hub.
 231   for (bus, dev) in reversed(sorted(usb_devices)):
 232     # Can not restart root usb connections
 233     if dev != '001':
 234       return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
 235       if return_code:
 236         logging.error('Error restarting USB device /dev/bus/usb/%s/%s',
 237                       bus, dev)
 238         all_restarted = False
 239       else:
 240         logging.info('Restarted USB device /dev/bus/usb/%s/%s', bus, dev)
 241
 242   return all_restarted
 243
 244
 245 def KillAllAdb():
 246   def GetAllAdb():
 247     for p in psutil.process_iter():
 248       try:
 249         if 'adb' in p.name:
 250           yield p
 251       except (psutil.NoSuchProcess, psutil.AccessDenied):
 252         pass
 253
 254   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
 255     for p in GetAllAdb():
 256       try:
 257         logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
 258                      ' '.join(p.cmdline))
 259         p.send_signal(sig)
 260       except (psutil.NoSuchProcess, psutil.AccessDenied):
 261         pass
 262   for p in GetAllAdb():
 263     try:
 264       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
 265                     ' '.join(p.cmdline))
 266     except (psutil.NoSuchProcess, psutil.AccessDenied):
 267       pass
 268
 269
 270 def main():
 271   parser = optparse.OptionParser()
 272   parser.add_option('', '--out-dir',
 273                     help='Directory where the device path is stored',
 274                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 275   parser.add_option('--no-provisioning-check', action='store_true',
 276                     help='Will not check if devices are provisioned properly.')
 277   parser.add_option('--device-status-dashboard', action='store_true',
 278                     help='Output device status data for dashboard.')
 279   parser.add_option('--restart-usb', action='store_true',
 280                     help='Restart USB ports before running device check.')
 281   parser.add_option('--json-output',
 282                     help='Output JSON information into a specified file.')
 283   parser.add_option('-v', '--verbose', action='count', default=1,
 284                     help='Log more information.')
 285
 286   options, args = parser.parse_args()
 287   if args:
 288     parser.error('Unknown options %s' % args)
 289
 290   run_tests_helper.SetLogLevel(options.verbose)
 291
 292   # Remove the last build's "bad devices" before checking device statuses.
 293   device_blacklist.ResetBlacklist()
 294
 295   try:
 296     expected_devices = device_list.GetPersistentDeviceList(
 297         os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
 298   except IOError:
 299     expected_devices = []
 300   devices = android_commands.GetAttachedDevices()
 301   # Only restart usb if devices are missing.
 302   if set(expected_devices) != set(devices):
 303     logging.warning('expected_devices: %s', expected_devices)
 304     logging.warning('devices: %s', devices)
 305     KillAllAdb()
 306     retries = 5
 307     usb_restarted = True
 308     if options.restart_usb:
 309       if not RestartUsb():
 310         usb_restarted = False
 311         bb_annotations.PrintWarning()
 312         logging.error('USB reset stage failed, '
 313                       'wait for any device to come back.')
 314     while retries:
 315       logging.info('retry adb devices...')
 316       time.sleep(1)
 317       devices = android_commands.GetAttachedDevices()
 318       if set(expected_devices) == set(devices):
 319         # All devices are online, keep going.
 320         break
 321       if not usb_restarted and devices:
 322         # The USB wasn't restarted, but there's at least one device online.
 323         # No point in trying to wait for all devices.
 324         break
 325       retries -= 1
 326
 327   # TODO(navabi): Test to make sure this fails and then fix call
 328   offline_devices = android_commands.GetAttachedDevices(
 329       hardware=False, emulator=False, offline=True)
 330
 331   types, builds, batteries, errors, devices_ok, json_data = (
 332       [], [], [], [], [], [])
 333   if devices:
 334     types, builds, batteries, errors, devices_ok, json_data = (
 335         zip(*[DeviceInfo(dev, options) for dev in devices]))
 336
 337   # Write device info to file for buildbot info display.
 338   if os.path.exists('/home/chrome-bot'):
 339     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
 340       for device in json_data:
 341         try:
 342           f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
 343               device['build'], float(device['battery']['temperature']) / 10,
 344               device['battery']['level']))
 345         except Exception:
 346           pass
 347
 348   err_msg = CheckForMissingDevices(options, devices) or []
 349
 350   unique_types = list(set(types))
 351   unique_builds = list(set(builds))
 352
 353   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 354                            % (len(devices), unique_types, unique_builds))
 355
 356   for j in json_data:
 357     logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
 358     logging.info('  Build: %s (%s)', j.get('build'), j.get('build_detail'))
 359     logging.info('  Current Battery Service state:')
 360     for k, v in j.get('battery', {}).iteritems():
 361       logging.info('    %s: %s', k, v)
 362     logging.info('  IMEI slice: %s', j.get('imei_slice'))
 363     logging.info('  WiFi IP: %s', j.get('wifi_ip'))
 364
 365
 366   for serial, dev_errors in zip(devices, errors):
 367     if dev_errors:
 368       err_msg += ['%s errors:' % serial]
 369       err_msg += ['    %s' % error for error in dev_errors]
 370
 371   if err_msg:
 372     bb_annotations.PrintWarning()
 373     for e in err_msg:
 374       logging.error(e)
 375     from_address = 'buildbot@chromium.org'
 376     to_addresses = ['chromium-android-device-alerts@google.com']
 377     bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 378     slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 379     subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 380     SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
 381
 382   if options.device_status_dashboard:
 383     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 384                                               [len(devices)], 'devices')
 385     perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 386                                               [len(offline_devices)], 'devices',
 387                                               'unimportant')
 388     for serial, battery in zip(devices, batteries):
 389       perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
 390                                                 [battery], '%',
 391                                                 'unimportant')
 392
 393   if options.json_output:
 394     with open(options.json_output, 'wb') as f:
 395       f.write(json.dumps(json_data, indent=4))
 396
 397   num_failed_devs = 0
 398   for device_ok, device in zip(devices_ok, devices):
 399     if not device_ok:
 400       logging.warning('Blacklisting %s', str(device))
 401       device_blacklist.ExtendBlacklist([str(device)])
 402       num_failed_devs += 1
 403
 404   if num_failed_devs == len(devices):
 405     return 2
 406
 407   if not devices:
 408     return 1
 409
 410
 411 if __name__ == '__main__':
 412   sys.exit(main())