tools/deep_memory_profiler/lib/sorter.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import cStringIO
   6 import json
   7 import logging
   8 import os
   9 import re
  10
  11 from lib.ordered_dict import OrderedDict
  12
  13
  14 LOGGER = logging.getLogger('dmprof')
  15
  16 BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  17
  18 DEFAULT_SORTERS = [
  19     os.path.join(BASE_PATH, 'sorters', 'malloc.browser-module.json'),
  20     os.path.join(BASE_PATH, 'sorters', 'malloc.renderer-module.json'),
  21     os.path.join(BASE_PATH, 'sorters', 'malloc.type.json'),
  22     os.path.join(BASE_PATH, 'sorters', 'malloc.WebCore.json'),
  23     os.path.join(BASE_PATH, 'sorters', 'vm.Android-specific.json'),
  24     os.path.join(BASE_PATH, 'sorters', 'vm.base.json'),
  25     os.path.join(BASE_PATH, 'sorters', 'vm.GPU.json'),
  26     os.path.join(BASE_PATH, 'sorters', 'vm.sharing.json'),
  27     os.path.join(BASE_PATH, 'sorters', 'vm.Skia.json'),
  28     os.path.join(BASE_PATH, 'sorters', 'vm.V8.json'),
  29     ]
  30
  31 DEFAULT_TEMPLATES = os.path.join(BASE_PATH, 'templates.json')
  32
  33
  34 class Unit(object):
  35   """Represents a minimum unit of memory usage categorization.
  36
  37   It is supposed to be inherited for some different spaces like the entire
  38   virtual memory and malloc arena. Such different spaces are called "worlds"
  39   in dmprof. (For example, the "vm" world and the "malloc" world.)
  40   """
  41   def __init__(self, unit_id, size):
  42     self._unit_id = unit_id
  43     self._size = size
  44
  45   @property
  46   def unit_id(self):
  47     return self._unit_id
  48
  49   @property
  50   def size(self):
  51     return self._size
  52
  53
  54 class VMUnit(Unit):
  55   """Represents a Unit for a memory region on virtual memory."""
  56   def __init__(self, unit_id, committed, reserved, mmap, region,
  57                pageframe=None, group_pfn_counts=None):
  58     super(VMUnit, self).__init__(unit_id, committed)
  59     self._reserved = reserved
  60     self._mmap = mmap
  61     self._region = region
  62     self._pageframe = pageframe
  63     self._group_pfn_counts = group_pfn_counts
  64
  65   @property
  66   def committed(self):
  67     return self._size
  68
  69   @property
  70   def reserved(self):
  71     return self._reserved
  72
  73   @property
  74   def mmap(self):
  75     return self._mmap
  76
  77   @property
  78   def region(self):
  79     return self._region
  80
  81   @property
  82   def pageframe(self):
  83     return self._pageframe
  84
  85   @property
  86   def group_pfn_counts(self):
  87     return self._group_pfn_counts
  88
  89
  90 class MMapUnit(VMUnit):
  91   """Represents a Unit for a mmap'ed region."""
  92   def __init__(self, unit_id, committed, reserved, region, bucket_set,
  93                pageframe=None, group_pfn_counts=None):
  94     super(MMapUnit, self).__init__(unit_id, committed, reserved, True,
  95                                    region, pageframe, group_pfn_counts)
  96     self._bucket_set = bucket_set
  97
  98   def __repr__(self):
  99     return str(self.region)
 100
 101   @property
 102   def bucket_set(self):
 103     return self._bucket_set
 104
 105
 106 class UnhookedUnit(VMUnit):
 107   """Represents a Unit for a non-mmap'ed memory region on virtual memory."""
 108   def __init__(self, unit_id, committed, reserved, region,
 109                pageframe=None, group_pfn_counts=None):
 110     super(UnhookedUnit, self).__init__(unit_id, committed, reserved, False,
 111                                        region, pageframe, group_pfn_counts)
 112
 113   def __repr__(self):
 114     return str(self.region)
 115
 116
 117 class MallocUnit(Unit):
 118   """Represents a Unit for a malloc'ed memory block."""
 119   def __init__(self, unit_id, size, alloc_count, free_count, bucket):
 120     super(MallocUnit, self).__init__(unit_id, size)
 121     self._bucket = bucket
 122     self._alloc_count = alloc_count
 123     self._free_count = free_count
 124
 125   def __repr__(self):
 126     return str(self.bucket)
 127
 128   @property
 129   def bucket(self):
 130     return self._bucket
 131
 132   @property
 133   def alloc_count(self):
 134     return self._alloc_count
 135
 136   @property
 137   def free_count(self):
 138     return self._free_count
 139
 140
 141 class UnitSet(object):
 142   """Represents an iterable set of Units."""
 143   def __init__(self, world):
 144     self._units = {}
 145     self._world = world
 146
 147   def __repr__(self):
 148     return str(self._units)
 149
 150   def __iter__(self):
 151     for unit_id in sorted(self._units):
 152       yield self._units[unit_id]
 153
 154   def append(self, unit, overwrite=False):
 155     if not overwrite and unit.unit_id in self._units:
 156       LOGGER.error('The unit id=%s already exists.' % str(unit.unit_id))
 157     self._units[unit.unit_id] = unit
 158
 159
 160 class AbstractRule(object):
 161   """An abstract class for rules to be matched with units."""
 162   def __init__(self, dct):
 163     self._name = dct['name']
 164     self._hidden = dct.get('hidden', False)
 165     self._subs = dct.get('subs', [])
 166
 167   def match(self, unit):
 168     raise NotImplementedError()
 169
 170   @property
 171   def name(self):
 172     return self._name
 173
 174   @property
 175   def hidden(self):
 176     return self._hidden
 177
 178   def iter_subs(self):
 179     for sub in self._subs:
 180       yield sub
 181
 182
 183 class VMRule(AbstractRule):
 184   """Represents a Rule to match with virtual memory regions."""
 185   def __init__(self, dct):
 186     super(VMRule, self).__init__(dct)
 187     self._backtrace_function = dct.get('backtrace_function', None)
 188     if self._backtrace_function:
 189       self._backtrace_function = re.compile(self._backtrace_function)
 190     self._backtrace_sourcefile = dct.get('backtrace_sourcefile', None)
 191     if self._backtrace_sourcefile:
 192       self._backtrace_sourcefile = re.compile(self._backtrace_sourcefile)
 193     self._mmap = dct.get('mmap', None)
 194     self._sharedwith = dct.get('sharedwith', [])
 195     self._mapped_pathname = dct.get('mapped_pathname', None)
 196     if self._mapped_pathname:
 197       self._mapped_pathname = re.compile(self._mapped_pathname)
 198     self._mapped_permission = dct.get('mapped_permission', None)
 199     if self._mapped_permission:
 200       self._mapped_permission = re.compile(self._mapped_permission)
 201
 202   def __repr__(self):
 203     result = cStringIO.StringIO()
 204     result.write('%s: ' % self._name)
 205     attributes = []
 206     attributes.append('mmap: %s' % self._mmap)
 207     if self._backtrace_function:
 208       attributes.append('backtrace_function: "%s"' %
 209                         self._backtrace_function.pattern)
 210     if self._sharedwith:
 211       attributes.append('sharedwith: "%s"' % self._sharedwith)
 212     if self._mapped_pathname:
 213       attributes.append('mapped_pathname: "%s"' % self._mapped_pathname.pattern)
 214     if self._mapped_permission:
 215       attributes.append('mapped_permission: "%s"' %
 216                         self._mapped_permission.pattern)
 217     result.write('{ %s }' % ', '.join(attributes))
 218     return result.getvalue()
 219
 220   def match(self, unit):
 221     if unit.mmap:
 222       assert unit.region[0] == 'hooked'
 223       bucket = unit.bucket_set.get(unit.region[1]['bucket_id'])
 224       assert bucket
 225       assert bucket.allocator_type == 'mmap'
 226
 227       stackfunction = bucket.symbolized_joined_stackfunction
 228       stacksourcefile = bucket.symbolized_joined_stacksourcefile
 229
 230       # TODO(dmikurube): Support shared memory.
 231       sharedwith = None
 232
 233       if self._mmap == False: # (self._mmap == None) should go through.
 234         return False
 235       if (self._backtrace_function and
 236           not self._backtrace_function.match(stackfunction)):
 237         return False
 238       if (self._backtrace_sourcefile and
 239           not self._backtrace_sourcefile.match(stacksourcefile)):
 240         return False
 241       if (self._mapped_pathname and
 242           not self._mapped_pathname.match(unit.region[1]['vma']['name'])):
 243         return False
 244       if (self._mapped_permission and
 245           not self._mapped_permission.match(
 246               unit.region[1]['vma']['readable'] +
 247               unit.region[1]['vma']['writable'] +
 248               unit.region[1]['vma']['executable'] +
 249               unit.region[1]['vma']['private'])):
 250         return False
 251       if (self._sharedwith and
 252           unit.pageframe and sharedwith not in self._sharedwith):
 253         return False
 254
 255       return True
 256
 257     else:
 258       assert unit.region[0] == 'unhooked'
 259
 260       # TODO(dmikurube): Support shared memory.
 261       sharedwith = None
 262
 263       if self._mmap == True: # (self._mmap == None) should go through.
 264         return False
 265       if (self._mapped_pathname and
 266           not self._mapped_pathname.match(unit.region[1]['vma']['name'])):
 267         return False
 268       if (self._mapped_permission and
 269           not self._mapped_permission.match(
 270               unit.region[1]['vma']['readable'] +
 271               unit.region[1]['vma']['writable'] +
 272               unit.region[1]['vma']['executable'] +
 273               unit.region[1]['vma']['private'])):
 274         return False
 275       if (self._sharedwith and
 276           unit.pageframe and sharedwith not in self._sharedwith):
 277         return False
 278
 279       return True
 280
 281
 282 class MallocRule(AbstractRule):
 283   """Represents a Rule to match with malloc'ed blocks."""
 284   def __init__(self, dct):
 285     super(MallocRule, self).__init__(dct)
 286     self._backtrace_function = dct.get('backtrace_function', None)
 287     if self._backtrace_function:
 288       self._backtrace_function = re.compile(self._backtrace_function)
 289     self._backtrace_sourcefile = dct.get('backtrace_sourcefile', None)
 290     if self._backtrace_sourcefile:
 291       self._backtrace_sourcefile = re.compile(self._backtrace_sourcefile)
 292     self._typeinfo = dct.get('typeinfo', None)
 293     if self._typeinfo:
 294       self._typeinfo = re.compile(self._typeinfo)
 295
 296   def __repr__(self):
 297     result = cStringIO.StringIO()
 298     result.write('%s: ' % self._name)
 299     attributes = []
 300     if self._backtrace_function:
 301       attributes.append('backtrace_function: "%s"' %
 302                         self._backtrace_function.pattern)
 303     if self._typeinfo:
 304       attributes.append('typeinfo: "%s"' % self._typeinfo.pattern)
 305     result.write('{ %s }' % ', '.join(attributes))
 306     return result.getvalue()
 307
 308   def match(self, unit):
 309     assert unit.bucket.allocator_type == 'malloc'
 310
 311     stackfunction = unit.bucket.symbolized_joined_stackfunction
 312     stacksourcefile = unit.bucket.symbolized_joined_stacksourcefile
 313     typeinfo = unit.bucket.symbolized_typeinfo
 314     if typeinfo.startswith('0x'):
 315       typeinfo = unit.bucket.typeinfo_name
 316
 317     return ((not self._backtrace_function or
 318              self._backtrace_function.match(stackfunction)) and
 319             (not self._backtrace_sourcefile or
 320              self._backtrace_sourcefile.match(stacksourcefile)) and
 321             (not self._typeinfo or self._typeinfo.match(typeinfo)))
 322
 323
 324 class AbstractSorter(object):
 325   """An abstract class for classifying Units with a set of Rules."""
 326   def __init__(self, dct):
 327     self._type = 'sorter'
 328     self._version = dct['version']
 329     self._world = dct['world']
 330     self._name = dct['name']
 331     self._root = dct.get('root', False)
 332     self._order = dct['order']
 333
 334     self._rules = []
 335     for rule in dct['rules']:
 336       if dct['world'] == 'vm':
 337         self._rules.append(VMRule(rule))
 338       elif dct['world'] == 'malloc':
 339         self._rules.append(MallocRule(rule))
 340       else:
 341         LOGGER.error('Unknown sorter world type')
 342
 343   def __repr__(self):
 344     result = cStringIO.StringIO()
 345     print >> result, '%s' % self._name
 346     print >> result, 'world=%s' % self._world
 347     print >> result, 'name=%s' % self._name
 348     print >> result, 'order=%s' % self._order
 349     print >> result, 'rules:'
 350     for rule in self._rules:
 351       print >> result, '  %s' % rule
 352     return result.getvalue()
 353
 354   @staticmethod
 355   def load(filename):
 356     with open(filename) as sorter_f:
 357       sorter_dict = json.load(sorter_f, object_pairs_hook=OrderedDict)
 358     if sorter_dict['world'] == 'vm':
 359       return VMSorter(sorter_dict)
 360     elif sorter_dict['world'] == 'malloc':
 361       return MallocSorter(sorter_dict)
 362     else:
 363       LOGGER.error('Unknown sorter world type')
 364       return None
 365
 366   @property
 367   def world(self):
 368     return self._world
 369
 370   @property
 371   def name(self):
 372     return self._name
 373
 374   @property
 375   def root(self):
 376     return self._root
 377
 378   def iter_rule(self):
 379     for rule in self._rules:
 380       yield rule
 381
 382   def find(self, unit):
 383     raise NotImplementedError()
 384
 385   def find_rule(self, name):
 386     """Finds a rule whose name is |name|. """
 387     for rule in self._rules:
 388       if rule.name == name:
 389         return rule
 390     return None
 391
 392
 393 class VMSorter(AbstractSorter):
 394   """Represents a Sorter for memory regions on virtual memory."""
 395   def __init__(self, dct):
 396     assert dct['world'] == 'vm'
 397     super(VMSorter, self).__init__(dct)
 398
 399   def find(self, unit):
 400     for rule in self._rules:
 401       if rule.match(unit):
 402         return rule
 403     return None
 404
 405
 406 class MallocSorter(AbstractSorter):
 407   """Represents a Sorter for malloc'ed blocks."""
 408   def __init__(self, dct):
 409     assert dct['world'] == 'malloc'
 410     super(MallocSorter, self).__init__(dct)
 411
 412   def find(self, unit):
 413     if not unit.bucket:
 414       return None
 415     assert unit.bucket.allocator_type == 'malloc'
 416
 417     # TODO(dmikurube): Utilize component_cache again, or remove it.
 418
 419     for rule in self._rules:
 420       if rule.match(unit):
 421         return rule
 422     return None
 423
 424
 425 class SorterTemplates(object):
 426   """Represents a template for sorters."""
 427   def __init__(self, dct):
 428     self._dict = dct
 429
 430   def as_dict(self):
 431     return self._dict
 432
 433   @staticmethod
 434   def load(filename):
 435     with open(filename) as templates_f:
 436       templates_dict = json.load(templates_f, object_pairs_hook=OrderedDict)
 437     return SorterTemplates(templates_dict)
 438
 439
 440 class SorterSet(object):
 441   """Represents an iterable set of Sorters."""
 442   def __init__(self, additional=None, default=None):
 443     if not additional:
 444       additional = []
 445     if not default:
 446       default = DEFAULT_SORTERS
 447     self._sorters = {}
 448     LOGGER.info('Loading sorters.')
 449     for filename in default + additional:
 450       LOGGER.info('  Loading a sorter "%s".' % filename)
 451       sorter = AbstractSorter.load(filename)
 452       if sorter.world not in self._sorters:
 453         self._sorters[sorter.world] = []
 454       self._sorters[sorter.world].append(sorter)
 455     self._templates = SorterTemplates.load(DEFAULT_TEMPLATES)
 456
 457   def __repr__(self):
 458     result = cStringIO.StringIO()
 459     for world, sorters in self._sorters.iteritems():
 460       for sorter in sorters:
 461         print >> result, '%s: %s' % (world, sorter)
 462     return result.getvalue()
 463
 464   def __iter__(self):
 465     for sorters in self._sorters.itervalues():
 466       for sorter in sorters:
 467         yield sorter
 468
 469   def iter_world(self, world):
 470     for sorter in self._sorters.get(world, []):
 471       yield sorter
 472
 473   @property
 474   def templates(self):
 475     return self._templates