build/android/tombstones.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6 #
   7 # Find the most recent tombstone file(s) on all connected devices
   8 # and prints their stacks.
   9 #
  10 # Assumes tombstone file was created with current symbols.
  11
  12 import datetime
  13 import itertools
  14 import logging
  15 import multiprocessing
  16 import os
  17 import re
  18 import subprocess
  19 import sys
  20 import optparse
  21
  22 from pylib.device import adb_wrapper
  23 from pylib.device import device_blacklist
  24 from pylib.device import device_errors
  25 from pylib.device import device_utils
  26 from pylib.utils import run_tests_helper
  27
  28
  29 _TZ_UTC = {'TZ': 'UTC'}
  30
  31 def _ListTombstones(device):
  32   """List the tombstone files on the device.
  33
  34   Args:
  35     device: An instance of DeviceUtils.
  36
  37   Yields:
  38     Tuples of (tombstone filename, date time of file on device).
  39   """
  40   try:
  41     lines = device.RunShellCommand(
  42         ['ls', '-a', '-l', '/data/tombstones'],
  43         as_root=True, check_return=True, env=_TZ_UTC, timeout=60)
  44     for line in lines:
  45       if 'tombstone' in line and not 'No such file or directory' in line:
  46         details = line.split()
  47         t = datetime.datetime.strptime(details[-3] + ' ' + details[-2],
  48                                        '%Y-%m-%d %H:%M')
  49         yield details[-1], t
  50   except device_errors.CommandFailedError:
  51     logging.exception('Could not retrieve tombstones.')
  52   except device_errors.CommandTimeoutError:
  53     logging.exception('Timed out retrieving tombstones.')
  54
  55
  56 def _GetDeviceDateTime(device):
  57   """Determine the date time on the device.
  58
  59   Args:
  60     device: An instance of DeviceUtils.
  61
  62   Returns:
  63     A datetime instance.
  64   """
  65   device_now_string = device.RunShellCommand(
  66       ['date'], check_return=True, env=_TZ_UTC)
  67   return datetime.datetime.strptime(
  68       device_now_string[0], '%a %b %d %H:%M:%S %Z %Y')
  69
  70
  71 def _GetTombstoneData(device, tombstone_file):
  72   """Retrieve the tombstone data from the device
  73
  74   Args:
  75     device: An instance of DeviceUtils.
  76     tombstone_file: the tombstone to retrieve
  77
  78   Returns:
  79     A list of lines
  80   """
  81   return device.ReadFile(
  82       '/data/tombstones/' + tombstone_file, as_root=True).splitlines()
  83
  84
  85 def _EraseTombstone(device, tombstone_file):
  86   """Deletes a tombstone from the device.
  87
  88   Args:
  89     device: An instance of DeviceUtils.
  90     tombstone_file: the tombstone to delete.
  91   """
  92   return device.RunShellCommand(
  93       ['rm', '/data/tombstones/' + tombstone_file],
  94       as_root=True, check_return=True)
  95
  96
  97 def _DeviceAbiToArch(device_abi):
  98   # The order of this list is significant to find the more specific match (e.g.,
  99   # arm64) before the less specific (e.g., arm).
 100   arches = ['arm64', 'arm', 'x86_64', 'x86_64', 'x86', 'mips']
 101   for arch in arches:
 102     if arch in device_abi:
 103       return arch
 104   raise RuntimeError('Unknown device ABI: %s' % device_abi)
 105
 106 def _ResolveSymbols(tombstone_data, include_stack, device_abi):
 107   """Run the stack tool for given tombstone input.
 108
 109   Args:
 110     tombstone_data: a list of strings of tombstone data.
 111     include_stack: boolean whether to include stack data in output.
 112     device_abi: the default ABI of the device which generated the tombstone.
 113
 114   Yields:
 115     A string for each line of resolved stack output.
 116   """
 117   # Check if the tombstone data has an ABI listed, if so use this in preference
 118   # to the device's default ABI.
 119   for line in tombstone_data:
 120     found_abi = re.search('ABI: \'(.+?)\'', line)
 121     if found_abi:
 122       device_abi = found_abi.group(1)
 123   arch = _DeviceAbiToArch(device_abi)
 124   if not arch:
 125     return
 126
 127   stack_tool = os.path.join(os.path.dirname(__file__), '..', '..',
 128                             'third_party', 'android_platform', 'development',
 129                             'scripts', 'stack')
 130   proc = subprocess.Popen([stack_tool, '--arch', arch], stdin=subprocess.PIPE,
 131                           stdout=subprocess.PIPE)
 132   output = proc.communicate(input='\n'.join(tombstone_data))[0]
 133   for line in output.split('\n'):
 134     if not include_stack and 'Stack Data:' in line:
 135       break
 136     yield line
 137
 138
 139 def _ResolveTombstone(tombstone):
 140   lines = []
 141   lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) +
 142             ', about this long ago: ' +
 143             (str(tombstone['device_now'] - tombstone['time']) +
 144             ' Device: ' + tombstone['serial'])]
 145   logging.info('\n'.join(lines))
 146   logging.info('Resolving...')
 147   lines += _ResolveSymbols(tombstone['data'], tombstone['stack'],
 148                            tombstone['device_abi'])
 149   return lines
 150
 151
 152 def _ResolveTombstones(jobs, tombstones):
 153   """Resolve a list of tombstones.
 154
 155   Args:
 156     jobs: the number of jobs to use with multiprocess.
 157     tombstones: a list of tombstones.
 158   """
 159   if not tombstones:
 160     logging.warning('No tombstones to resolve.')
 161     return
 162   if len(tombstones) == 1:
 163     data = [_ResolveTombstone(tombstones[0])]
 164   else:
 165     pool = multiprocessing.Pool(processes=jobs)
 166     data = pool.map(_ResolveTombstone, tombstones)
 167   for tombstone in data:
 168     for line in tombstone:
 169       logging.info(line)
 170
 171
 172 def _GetTombstonesForDevice(device, options):
 173   """Returns a list of tombstones on a given device.
 174
 175   Args:
 176     device: An instance of DeviceUtils.
 177     options: command line arguments from OptParse
 178   """
 179   ret = []
 180   all_tombstones = list(_ListTombstones(device))
 181   if not all_tombstones:
 182     logging.warning('No tombstones.')
 183     return ret
 184
 185   # Sort the tombstones in date order, descending
 186   all_tombstones.sort(cmp=lambda a, b: cmp(b[1], a[1]))
 187
 188   # Only resolve the most recent unless --all-tombstones given.
 189   tombstones = all_tombstones if options.all_tombstones else [all_tombstones[0]]
 190
 191   device_now = _GetDeviceDateTime(device)
 192   try:
 193     for tombstone_file, tombstone_time in tombstones:
 194       ret += [{'serial': str(device),
 195                'device_abi': device.product_cpu_abi,
 196                'device_now': device_now,
 197                'time': tombstone_time,
 198                'file': tombstone_file,
 199                'stack': options.stack,
 200                'data': _GetTombstoneData(device, tombstone_file)}]
 201   except device_errors.CommandFailedError:
 202     for line in device.RunShellCommand(
 203         ['ls', '-a', '-l', '/data/tombstones'],
 204         as_root=True, check_return=True, env=_TZ_UTC, timeout=60):
 205       logging.info('%s: %s', str(device), line)
 206     raise
 207
 208   # Erase all the tombstones if desired.
 209   if options.wipe_tombstones:
 210     for tombstone_file, _ in all_tombstones:
 211       _EraseTombstone(device, tombstone_file)
 212
 213   return ret
 214
 215
 216 def main():
 217   custom_handler = logging.StreamHandler(sys.stdout)
 218   custom_handler.setFormatter(run_tests_helper.CustomFormatter())
 219   logging.getLogger().addHandler(custom_handler)
 220   logging.getLogger().setLevel(logging.INFO)
 221
 222   parser = optparse.OptionParser()
 223   parser.add_option('--device',
 224                     help='The serial number of the device. If not specified '
 225                          'will use all devices.')
 226   parser.add_option('--blacklist-file', help='Device blacklist JSON file.')
 227   parser.add_option('-a', '--all-tombstones', action='store_true',
 228                     help="""Resolve symbols for all tombstones, rather than just
 229                          the most recent""")
 230   parser.add_option('-s', '--stack', action='store_true',
 231                     help='Also include symbols for stack data')
 232   parser.add_option('-w', '--wipe-tombstones', action='store_true',
 233                     help='Erase all tombstones from device after processing')
 234   parser.add_option('-j', '--jobs', type='int',
 235                     default=4,
 236                     help='Number of jobs to use when processing multiple '
 237                          'crash stacks.')
 238   options, _ = parser.parse_args()
 239
 240   if options.blacklist_file:
 241     blacklist = device_blacklist.Blacklist(options.blacklist_file)
 242   else:
 243     blacklist = None
 244
 245   if options.device:
 246     devices = [device_utils.DeviceUtils(options.device)]
 247   else:
 248     devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
 249
 250   # This must be done serially because strptime can hit a race condition if
 251   # used for the first time in a multithreaded environment.
 252   # http://bugs.python.org/issue7980
 253   tombstones = []
 254   for device in devices:
 255     tombstones += _GetTombstonesForDevice(device, options)
 256
 257   _ResolveTombstones(options.jobs, tombstones)
 258
 259
 260 if __name__ == '__main__':
 261   sys.exit(main())