tools/deep_memory_profiler/lib/policy.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import logging
   7 import os
   8 import re
   9
  10
  11 LOGGER = logging.getLogger('dmprof')
  12
  13 BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  14 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json')
  15
  16 # Heap Profile Policy versions
  17
  18 # POLICY_DEEP_1 DOES NOT include allocation_type columns.
  19 # mmap regions are distincted w/ mmap frames in the pattern column.
  20 POLICY_DEEP_1 = 'POLICY_DEEP_1'
  21
  22 # POLICY_DEEP_2 DOES include allocation_type columns.
  23 # mmap regions are distincted w/ the allocation_type column.
  24 POLICY_DEEP_2 = 'POLICY_DEEP_2'
  25
  26 # POLICY_DEEP_3 is in JSON format.
  27 POLICY_DEEP_3 = 'POLICY_DEEP_3'
  28
  29 # POLICY_DEEP_3 contains typeinfo.
  30 POLICY_DEEP_4 = 'POLICY_DEEP_4'
  31
  32
  33 class Rule(object):
  34   """Represents one matching rule in a policy file."""
  35
  36   def __init__(self,
  37                name,
  38                allocator_type,
  39                stackfunction_pattern=None,
  40                stacksourcefile_pattern=None,
  41                typeinfo_pattern=None,
  42                mappedpathname_pattern=None,
  43                mappedpermission_pattern=None,
  44                sharedwith=None):
  45     self._name = name
  46     self._allocator_type = allocator_type
  47
  48     self._stackfunction_pattern = None
  49     if stackfunction_pattern:
  50       self._stackfunction_pattern = re.compile(
  51           stackfunction_pattern + r'\Z')
  52
  53     self._stacksourcefile_pattern = None
  54     if stacksourcefile_pattern:
  55       self._stacksourcefile_pattern = re.compile(
  56           stacksourcefile_pattern + r'\Z')
  57
  58     self._typeinfo_pattern = None
  59     if typeinfo_pattern:
  60       self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z')
  61
  62     self._mappedpathname_pattern = None
  63     if mappedpathname_pattern:
  64       self._mappedpathname_pattern = re.compile(mappedpathname_pattern + r'\Z')
  65
  66     self._mappedpermission_pattern = None
  67     if mappedpermission_pattern:
  68       self._mappedpermission_pattern = re.compile(
  69           mappedpermission_pattern + r'\Z')
  70
  71     self._sharedwith = []
  72     if sharedwith:
  73       self._sharedwith = sharedwith
  74
  75   @property
  76   def name(self):
  77     return self._name
  78
  79   @property
  80   def allocator_type(self):
  81     return self._allocator_type
  82
  83   @property
  84   def stackfunction_pattern(self):
  85     return self._stackfunction_pattern
  86
  87   @property
  88   def stacksourcefile_pattern(self):
  89     return self._stacksourcefile_pattern
  90
  91   @property
  92   def typeinfo_pattern(self):
  93     return self._typeinfo_pattern
  94
  95   @property
  96   def mappedpathname_pattern(self):
  97     return self._mappedpathname_pattern
  98
  99   @property
 100   def mappedpermission_pattern(self):
 101     return self._mappedpermission_pattern
 102
 103   @property
 104   def sharedwith(self):
 105     return self._sharedwith
 106
 107
 108 class Policy(object):
 109   """Represents a policy, a content of a policy file."""
 110
 111   def __init__(self, rules, version, components):
 112     self._rules = rules
 113     self._version = version
 114     self._components = components
 115
 116   @property
 117   def rules(self):
 118     return self._rules
 119
 120   @property
 121   def version(self):
 122     return self._version
 123
 124   @property
 125   def components(self):
 126     return self._components
 127
 128   def find_rule(self, component_name, after_rule=None):
 129     """Finds a rule whose name is |component_name|.
 130
 131     If there are multiple rules with same component name,
 132     use |after_rule| to search the rule after it.
 133     """
 134     found_after_rule = False
 135     for rule in self._rules:
 136       if after_rule and not found_after_rule:
 137         if rule == after_rule:
 138           found_after_rule = True
 139         continue
 140
 141       if rule.name == component_name:
 142         return rule
 143     return None
 144
 145   def find_malloc(self, bucket):
 146     """Finds a matching component name which a given |bucket| belongs to.
 147
 148     Args:
 149         bucket: A Bucket object to be searched for.
 150
 151     Returns:
 152         A string representing a component name.
 153     """
 154     assert not bucket or bucket.allocator_type == 'malloc'
 155
 156     if not bucket:
 157       return 'no-bucket'
 158     if bucket.component_cache:
 159       return bucket.component_cache
 160
 161     stackfunction = bucket.symbolized_joined_stackfunction
 162     stacksourcefile = bucket.symbolized_joined_stacksourcefile
 163     typeinfo = bucket.symbolized_typeinfo
 164     if typeinfo.startswith('0x'):
 165       typeinfo = bucket.typeinfo_name
 166
 167     for rule in self._rules:
 168       if (rule.allocator_type == 'malloc' and
 169           (not rule.stackfunction_pattern or
 170            rule.stackfunction_pattern.match(stackfunction)) and
 171           (not rule.stacksourcefile_pattern or
 172            rule.stacksourcefile_pattern.match(stacksourcefile)) and
 173           (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))):
 174         bucket.component_cache = rule.name
 175         return rule.name
 176
 177     assert False
 178
 179   def find_mmap(self, region, bucket_set,
 180                 pageframe=None, group_pfn_counts=None):
 181     """Finds a matching component which a given mmap |region| belongs to.
 182
 183     It uses |bucket_set| to match with backtraces.  If |pageframe| is given,
 184     it considers memory sharing among processes.
 185
 186     NOTE: Don't use Bucket's |component_cache| for mmap regions because they're
 187     classified not only with bucket information (mappedpathname for example).
 188
 189     Args:
 190         region: A tuple representing a memory region.
 191         bucket_set: A BucketSet object to look up backtraces.
 192         pageframe: A PageFrame object representing a pageframe maybe including
 193             a pagecount.
 194         group_pfn_counts: A dict mapping a PFN to the number of times the
 195             the pageframe is mapped by the known "group (Chrome)" processes.
 196
 197     Returns:
 198         A string representing a component name.
 199     """
 200     assert region[0] == 'hooked'
 201     bucket = bucket_set.get(region[1]['bucket_id'])
 202     assert not bucket or bucket.allocator_type == 'mmap'
 203
 204     if not bucket:
 205       return 'no-bucket', None
 206
 207     stackfunction = bucket.symbolized_joined_stackfunction
 208     stacksourcefile = bucket.symbolized_joined_stacksourcefile
 209     sharedwith = self._categorize_pageframe(pageframe, group_pfn_counts)
 210
 211     for rule in self._rules:
 212       if (rule.allocator_type == 'mmap' and
 213           (not rule.stackfunction_pattern or
 214            rule.stackfunction_pattern.match(stackfunction)) and
 215           (not rule.stacksourcefile_pattern or
 216            rule.stacksourcefile_pattern.match(stacksourcefile)) and
 217           (not rule.mappedpathname_pattern or
 218            rule.mappedpathname_pattern.match(region[1]['vma']['name'])) and
 219           (not rule.mappedpermission_pattern or
 220            rule.mappedpermission_pattern.match(
 221                region[1]['vma']['readable'] +
 222                region[1]['vma']['writable'] +
 223                region[1]['vma']['executable'] +
 224                region[1]['vma']['private'])) and
 225           (not rule.sharedwith or
 226            not pageframe or sharedwith in rule.sharedwith)):
 227         return rule.name, bucket
 228
 229     assert False
 230
 231   def find_unhooked(self, region, pageframe=None, group_pfn_counts=None):
 232     """Finds a matching component which a given unhooked |region| belongs to.
 233
 234     If |pageframe| is given, it considers memory sharing among processes.
 235
 236     Args:
 237         region: A tuple representing a memory region.
 238         pageframe: A PageFrame object representing a pageframe maybe including
 239             a pagecount.
 240         group_pfn_counts: A dict mapping a PFN to the number of times the
 241             the pageframe is mapped by the known "group (Chrome)" processes.
 242
 243     Returns:
 244         A string representing a component name.
 245     """
 246     assert region[0] == 'unhooked'
 247     sharedwith = self._categorize_pageframe(pageframe, group_pfn_counts)
 248
 249     for rule in self._rules:
 250       if (rule.allocator_type == 'unhooked' and
 251           (not rule.mappedpathname_pattern or
 252            rule.mappedpathname_pattern.match(region[1]['vma']['name'])) and
 253           (not rule.mappedpermission_pattern or
 254            rule.mappedpermission_pattern.match(
 255                region[1]['vma']['readable'] +
 256                region[1]['vma']['writable'] +
 257                region[1]['vma']['executable'] +
 258                region[1]['vma']['private'])) and
 259           (not rule.sharedwith or
 260            not pageframe or sharedwith in rule.sharedwith)):
 261         return rule.name
 262
 263     assert False
 264
 265   @staticmethod
 266   def load(filename, filetype):
 267     """Loads a policy file of |filename| in a |format|.
 268
 269     Args:
 270         filename: A filename to be loaded.
 271         filetype: A string to specify a type of the file.  Only 'json' is
 272             supported for now.
 273
 274     Returns:
 275         A loaded Policy object.
 276     """
 277     with open(os.path.join(BASE_PATH, filename)) as policy_f:
 278       return Policy.parse(policy_f, filetype)
 279
 280   @staticmethod
 281   def parse(policy_f, filetype):
 282     """Parses a policy file content in a |format|.
 283
 284     Args:
 285         policy_f: An IO object to be loaded.
 286         filetype: A string to specify a type of the file.  Only 'json' is
 287             supported for now.
 288
 289     Returns:
 290         A loaded Policy object.
 291     """
 292     if filetype == 'json':
 293       return Policy._parse_json(policy_f)
 294     else:
 295       return None
 296
 297   JSON_COMMENT_REGEX = re.compile(r'//.*')
 298
 299   @staticmethod
 300   def _parse_json(policy_f):
 301     """Parses policy file in json format.
 302
 303     A policy file contains component's names and their stacktrace pattern
 304     written in regular expression.  Those patterns are matched against each
 305     symbols of each stacktraces in the order written in the policy file
 306
 307     Args:
 308          policy_f: A File/IO object to read.
 309
 310     Returns:
 311          A loaded policy object.
 312     """
 313     policy_json = policy_f.read()
 314     policy_json = re.sub(Policy.JSON_COMMENT_REGEX, '', policy_json)
 315     policy = json.loads(policy_json)
 316
 317     rules = []
 318     for rule in policy['rules']:
 319       stackfunction = rule.get('stackfunction') or rule.get('stacktrace')
 320       stacksourcefile = rule.get('stacksourcefile')
 321       rules.append(Rule(
 322           rule['name'],
 323           rule['allocator'],  # allocator_type
 324           stackfunction,
 325           stacksourcefile,
 326           rule['typeinfo'] if 'typeinfo' in rule else None,
 327           rule.get('mappedpathname'),
 328           rule.get('mappedpermission'),
 329           rule.get('sharedwith')))
 330
 331     return Policy(rules, policy['version'], policy['components'])
 332
 333   @staticmethod
 334   def _categorize_pageframe(pageframe, group_pfn_counts):
 335     """Categorizes a pageframe based on its sharing status.
 336
 337     Returns:
 338         'private' if |pageframe| is not shared with other processes.  'group'
 339         if |pageframe| is shared only with group (Chrome-related) processes.
 340         'others' if |pageframe| is shared with non-group processes.
 341     """
 342     if not pageframe:
 343       return 'private'
 344
 345     if pageframe.pagecount:
 346       if pageframe.pagecount == 1:
 347         return 'private'
 348       elif pageframe.pagecount <= group_pfn_counts.get(pageframe.pfn, 0) + 1:
 349         return 'group'
 350       else:
 351         return 'others'
 352     else:
 353       if pageframe.pfn in group_pfn_counts:
 354         return 'group'
 355       else:
 356         return 'private'
 357
 358
 359 class PolicySet(object):
 360   """Represents a set of policies."""
 361
 362   def __init__(self, policy_directory):
 363     self._policy_directory = policy_directory
 364
 365   @staticmethod
 366   def load(labels=None):
 367     """Loads a set of policies via the "default policy directory".
 368
 369     The "default policy directory" contains pairs of policies and their labels.
 370     For example, a policy "policy.l0.json" is labeled "l0" in the default
 371     policy directory "policies.json".
 372
 373     All policies in the directory are loaded by default.  Policies can be
 374     limited by |labels|.
 375
 376     Args:
 377         labels: An array that contains policy labels to be loaded.
 378
 379     Returns:
 380         A PolicySet object.
 381     """
 382     default_policy_directory = PolicySet._load_default_policy_directory()
 383     if labels:
 384       specified_policy_directory = {}
 385       for label in labels:
 386         if label in default_policy_directory:
 387           specified_policy_directory[label] = default_policy_directory[label]
 388         # TODO(dmikurube): Load an un-labeled policy file.
 389       return PolicySet._load_policies(specified_policy_directory)
 390     else:
 391       return PolicySet._load_policies(default_policy_directory)
 392
 393   def __len__(self):
 394     return len(self._policy_directory)
 395
 396   def __iter__(self):
 397     for label in self._policy_directory:
 398       yield label
 399
 400   def __getitem__(self, label):
 401     return self._policy_directory[label]
 402
 403   @staticmethod
 404   def _load_default_policy_directory():
 405     with open(POLICIES_JSON_PATH, mode='r') as policies_f:
 406       default_policy_directory = json.load(policies_f)
 407     return default_policy_directory
 408
 409   @staticmethod
 410   def _load_policies(directory):
 411     LOGGER.info('Loading policy files.')
 412     policies = {}
 413     for label in directory:
 414       LOGGER.info('  %s: %s' % (label, directory[label]['file']))
 415       loaded = Policy.load(directory[label]['file'], directory[label]['format'])
 416       if loaded:
 417         policies[label] = loaded
 418     return PolicySet(policies)