build/android/buildbot/bb_device_status_check.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """A class to keep track of devices across builds and report state."""
   8 import logging
   9 import optparse
  10 import os
  11 import smtplib
  12 import sys
  13 import re
  14 import urllib
  15
  16 import bb_annotations
  17
  18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  19 from pylib import android_commands
  20 from pylib import constants
  21 from pylib import perf_tests_helper
  22 from pylib.cmd_helper import GetCmdOutput
  23
  24
  25 def DeviceInfo(serial, options):
  26   """Gathers info on a device via various adb calls.
  27
  28   Args:
  29     serial: The serial of the attached device to construct info about.
  30
  31   Returns:
  32     Tuple of device type, build id, report as a string, error messages, and
  33     boolean indicating whether or not device can be used for testing.
  34   """
  35
  36   device_adb = android_commands.AndroidCommands(serial)
  37
  38   # TODO(navabi): Replace AdbShellCmd with device_adb.
  39   device_type = device_adb.GetBuildProduct()
  40   device_build = device_adb.GetBuildId()
  41   device_build_type = device_adb.GetBuildType()
  42   device_product_name = device_adb.GetProductName()
  43
  44   battery = device_adb.GetBatteryInfo()
  45
  46   def _GetData(re_expression, line, lambda_function=lambda x:x):
  47     if not line:
  48       return 'Unknown'
  49     found = re.findall(re_expression, line)
  50     if found and len(found):
  51       return lambda_function(found[0])
  52     return 'Unknown'
  53
  54   if options.device_status_dashboard:
  55     # Dashboard does not track install speed. Do not unnecessarily install.
  56     install_speed = 'Unknown'
  57   else:
  58     install_output = GetCmdOutput(
  59       ['%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT,
  60        '--apk',
  61        '%s/build/android/CheckInstallApk-debug.apk' % constants.DIR_SOURCE_ROOT
  62       ])
  63     install_speed = _GetData('(\d+) KB/s', install_output)
  64
  65   ac_power = _GetData('AC powered: (\w+)', battery)
  66   battery_level = _GetData('level: (\d+)', battery)
  67   battery_temp = _GetData('temperature: (\d+)', battery,
  68                           lambda x: float(x) / 10.0)
  69   imei_slice = _GetData('Device ID = (\d+)',
  70                         device_adb.GetSubscriberInfo(),
  71                         lambda x: x[-6:])
  72   report = ['Device %s (%s)' % (serial, device_type),
  73             '  Build: %s (%s)' %
  74               (device_build, device_adb.GetBuildFingerprint()),
  75             '  Battery: %s%%' % battery_level,
  76             '  Battery temp: %s' % battery_temp,
  77             '  IMEI slice: %s' % imei_slice,
  78             '  Wifi IP: %s' % device_adb.GetWifiIP(),
  79             '  Install Speed: %s KB/s' % install_speed,
  80             '']
  81
  82   errors = []
  83   if battery_level < 15:
  84     errors += ['Device critically low in battery. Turning off device.']
  85   if not options.no_provisioning_check:
  86     setup_wizard_disabled = device_adb.GetSetupWizardStatus() == 'DISABLED'
  87     if not setup_wizard_disabled and device_build_type != 'user':
  88       errors += ['Setup wizard not disabled. Was it provisioned correctly?']
  89   if device_product_name == 'mantaray' and ac_power != 'true':
  90     errors += ['Mantaray device not connected to AC power.']
  91   # TODO(navabi): Insert warning once we have a better handle of what install
  92   # speeds to expect. The following lines were causing too many alerts.
  93   # if install_speed < 500:
  94   #   errors += ['Device install speed too low. Do not use for testing.']
  95
  96   # Causing the device status check step fail for slow install speed or low
  97   # battery currently is too disruptive to the bots (especially try bots).
  98   # Turn off devices with low battery and the step does not fail.
  99   if battery_level < 15:
 100     device_adb.EnableAdbRoot()
 101     device_adb.Shutdown()
 102   full_report = '\n'.join(report)
 103   return device_type, device_build, battery_level, full_report, errors, True
 104
 105
 106 def CheckForMissingDevices(options, adb_online_devs):
 107   """Uses file of previous online devices to detect broken phones.
 108
 109   Args:
 110     options: out_dir parameter of options argument is used as the base
 111              directory to load and update the cache file.
 112     adb_online_devs: A list of serial numbers of the currently visible
 113                      and online attached devices.
 114   """
 115   # TODO(navabi): remove this once the bug that causes different number
 116   # of devices to be detected between calls is fixed.
 117   logger = logging.getLogger()
 118   logger.setLevel(logging.INFO)
 119
 120   out_dir = os.path.abspath(options.out_dir)
 121
 122   def ReadDeviceList(file_name):
 123     devices_path = os.path.join(out_dir, file_name)
 124     devices = []
 125     try:
 126       with open(devices_path) as f:
 127         devices = f.read().splitlines()
 128     except IOError:
 129       # Ignore error, file might not exist
 130       pass
 131     return devices
 132
 133   def WriteDeviceList(file_name, device_list):
 134     path = os.path.join(out_dir, file_name)
 135     if not os.path.exists(out_dir):
 136       os.makedirs(out_dir)
 137     with open(path, 'w') as f:
 138       # Write devices currently visible plus devices previously seen.
 139       f.write('\n'.join(set(device_list)))
 140
 141   last_devices_path = os.path.join(out_dir, '.last_devices')
 142   last_devices = ReadDeviceList('.last_devices')
 143   missing_devs = list(set(last_devices) - set(adb_online_devs))
 144
 145   all_known_devices = list(set(adb_online_devs) | set(last_devices))
 146   WriteDeviceList('.last_devices', all_known_devices)
 147   WriteDeviceList('.last_missing', missing_devs)
 148
 149   if not all_known_devices:
 150     # This can happen if for some reason the .last_devices file is not
 151     # present or if it was empty.
 152     return ['No online devices. Have any devices been plugged in?']
 153   if missing_devs:
 154     devices_missing_msg = '%d devices not detected.' % len(missing_devs)
 155     bb_annotations.PrintSummaryText(devices_missing_msg)
 156
 157     # TODO(navabi): Debug by printing both output from GetCmdOutput and
 158     # GetAttachedDevices to compare results.
 159     crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
 160                   '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
 161                   (urllib.quote('Device Offline'),
 162                    urllib.quote('Buildbot: %s %s\n'
 163                                 'Build: %s\n'
 164                                 '(please don\'t change any labels)' %
 165                                 (os.environ.get('BUILDBOT_BUILDERNAME'),
 166                                  os.environ.get('BUILDBOT_SLAVENAME'),
 167                                  os.environ.get('BUILDBOT_BUILDNUMBER')))))
 168     return ['Current online devices: %s' % adb_online_devs,
 169             '%s are no longer visible. Were they removed?\n' % missing_devs,
 170             'SHERIFF:\n',
 171             '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
 172             'Cache file: %s\n\n' % last_devices_path,
 173             'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
 174             'adb devices(GetAttachedDevices): %s' %
 175             android_commands.GetAttachedDevices()]
 176   else:
 177     new_devs = set(adb_online_devs) - set(last_devices)
 178     if new_devs and os.path.exists(last_devices_path):
 179       bb_annotations.PrintWarning()
 180       bb_annotations.PrintSummaryText(
 181           '%d new devices detected' % len(new_devs))
 182       print ('New devices detected %s. And now back to your '
 183              'regularly scheduled program.' % list(new_devs))
 184
 185
 186 def SendDeviceStatusAlert(msg):
 187   from_address = 'buildbot@chromium.org'
 188   to_address = 'chromium-android-device-alerts@google.com'
 189   bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
 190   slave_name = os.environ.get('BUILDBOT_SLAVENAME')
 191   subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
 192   msg_body = '\r\n'.join(['From: %s' % from_address, 'To: %s' % to_address,
 193                           'Subject: %s' % subject, '', msg])
 194   try:
 195     server = smtplib.SMTP('localhost')
 196     server.sendmail(from_address, [to_address], msg_body)
 197     server.quit()
 198   except Exception as e:
 199     print 'Failed to send alert email. Error: %s' % e
 200
 201
 202 def main():
 203   parser = optparse.OptionParser()
 204   parser.add_option('', '--out-dir',
 205                     help='Directory where the device path is stored',
 206                     default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
 207   parser.add_option('--no-provisioning-check', action='store_true',
 208                     help='Will not check if devices are provisioned properly.')
 209   parser.add_option('--device-status-dashboard', action='store_true',
 210                     help='Output device status data for dashboard.')
 211   options, args = parser.parse_args()
 212   if args:
 213     parser.error('Unknown options %s' % args)
 214   devices = android_commands.GetAttachedDevices()
 215   # TODO(navabi): Test to make sure this fails and then fix call
 216   offline_devices = android_commands.GetAttachedDevices(hardware=False,
 217                                                         emulator=False,
 218                                                         offline=True)
 219
 220   types, builds, batteries, reports, errors = [], [], [], [], []
 221   fail_step_lst = []
 222   if devices:
 223     types, builds, batteries, reports, errors, fail_step_lst = (
 224         zip(*[DeviceInfo(dev, options) for dev in devices]))
 225
 226   err_msg = CheckForMissingDevices(options, devices) or []
 227
 228   unique_types = list(set(types))
 229   unique_builds = list(set(builds))
 230
 231   bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
 232                            % (len(devices), unique_types, unique_builds))
 233   print '\n'.join(reports)
 234
 235   for serial, dev_errors in zip(devices, errors):
 236     if dev_errors:
 237       err_msg += ['%s errors:' % serial]
 238       err_msg += ['    %s' % error for error in dev_errors]
 239
 240   if err_msg:
 241     bb_annotations.PrintWarning()
 242     msg = '\n'.join(err_msg)
 243     print msg
 244     SendDeviceStatusAlert(msg)
 245
 246   if options.device_status_dashboard:
 247     perf_tests_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
 248                                       [len(devices)], 'devices')
 249     perf_tests_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
 250                                       [len(offline_devices)], 'devices',
 251                                       'unimportant')
 252     for serial, battery in zip(devices, batteries):
 253       perf_tests_helper.PrintPerfResult('DeviceBattery', serial, [battery], '%',
 254                                         'unimportant')
 255
 256   if False in fail_step_lst:
 257     # TODO(navabi): Build fails on device status check step if there exists any
 258     # devices with critically low battery or install speed. Remove those devices
 259     # from testing, allowing build to continue with good devices.
 260     return 1
 261
 262   if not devices:
 263     return 1
 264
 265
 266 if __name__ == '__main__':
 267   sys.exit(main())