tools/opt-viewer/optrecord.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import yaml
   6 # Try to use the C parser.
   7 try:
   8     from yaml import CLoader as Loader
   9 except ImportError:
  10     print("For faster parsing, you may want to install libYAML for PyYAML")
  11     from yaml import Loader
  12
  13 import cgi
  14 from collections import defaultdict
  15 import fnmatch
  16 import functools
  17 from multiprocessing import Lock
  18 import os, os.path
  19 import subprocess
  20 try:
  21     # The previously builtin function `intern()` was moved
  22     # to the `sys` module in Python 3.
  23     from sys import intern
  24 except:
  25     pass
  26
  27 import re
  28
  29 import optpmap
  30
  31 try:
  32     dict.iteritems
  33 except AttributeError:
  34     # Python 3
  35     def itervalues(d):
  36         return iter(d.values())
  37     def iteritems(d):
  38         return iter(d.items())
  39 else:
  40     # Python 2
  41     def itervalues(d):
  42         return d.itervalues()
  43     def iteritems(d):
  44         return d.iteritems()
  45
  46
  47 def html_file_name(filename):
  48     return filename.replace('/', '_').replace('#', '_') + ".html"
  49
  50
  51 def make_link(File, Line):
  52     return "\"{}#L{}\"".format(html_file_name(File), Line)
  53
  54
  55 class Remark(yaml.YAMLObject):
  56     # Work-around for http://pyyaml.org/ticket/154.
  57     yaml_loader = Loader
  58
  59     default_demangler = 'c++filt -n'
  60     demangler_proc = None
  61
  62     @classmethod
  63     def set_demangler(cls, demangler):
  64         cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  65         cls.demangler_lock = Lock()
  66
  67     @classmethod
  68     def demangle(cls, name):
  69         with cls.demangler_lock:
  70             cls.demangler_proc.stdin.write((name + '\n').encode('utf-8'))
  71             cls.demangler_proc.stdin.flush()
  72             return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8')
  73
  74     # Intern all strings since we have lot of duplication across filenames,
  75     # remark text.
  76     #
  77     # Change Args from a list of dicts to a tuple of tuples.  This saves
  78     # memory in two ways.  One, a small tuple is significantly smaller than a
  79     # small dict.  Two, using tuple instead of list allows Args to be directly
  80     # used as part of the key (in Python only immutable types are hashable).
  81     def _reduce_memory(self):
  82         self.Pass = intern(self.Pass)
  83         self.Name = intern(self.Name)
  84         try:
  85             # Can't intern unicode strings.
  86             self.Function = intern(self.Function)
  87         except:
  88             pass
  89
  90         def _reduce_memory_dict(old_dict):
  91             new_dict = dict()
  92             for (k, v) in iteritems(old_dict):
  93                 if type(k) is str:
  94                     k = intern(k)
  95
  96                 if type(v) is str:
  97                     v = intern(v)
  98                 elif type(v) is dict:
  99                     # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
 100                     v = _reduce_memory_dict(v)
 101                 new_dict[k] = v
 102             return tuple(new_dict.items())
 103
 104         self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
 105
 106     # The inverse operation of the dictonary-related memory optimization in
 107     # _reduce_memory_dict.  E.g.
 108     #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
 109     def recover_yaml_structure(self):
 110         def tuple_to_dict(t):
 111             d = dict()
 112             for (k, v) in t:
 113                 if type(v) is tuple:
 114                     v = tuple_to_dict(v)
 115                 d[k] = v
 116             return d
 117
 118         self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
 119
 120     def canonicalize(self):
 121         if not hasattr(self, 'Hotness'):
 122             self.Hotness = 0
 123         if not hasattr(self, 'Args'):
 124             self.Args = []
 125         self._reduce_memory()
 126
 127     @property
 128     def File(self):
 129         return self.DebugLoc['File']
 130
 131     @property
 132     def Line(self):
 133         return int(self.DebugLoc['Line'])
 134
 135     @property
 136     def Column(self):
 137         return self.DebugLoc['Column']
 138
 139     @property
 140     def DebugLocString(self):
 141         return "{}:{}:{}".format(self.File, self.Line, self.Column)
 142
 143     @property
 144     def DemangledFunctionName(self):
 145         return self.demangle(self.Function)
 146
 147     @property
 148     def Link(self):
 149         return make_link(self.File, self.Line)
 150
 151     def getArgString(self, mapping):
 152         mapping = dict(list(mapping))
 153         dl = mapping.get('DebugLoc')
 154         if dl:
 155             del mapping['DebugLoc']
 156
 157         assert(len(mapping) == 1)
 158         (key, value) = list(mapping.items())[0]
 159
 160         if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
 161             value = cgi.escape(self.demangle(value))
 162
 163         if dl and key != 'Caller':
 164             dl_dict = dict(list(dl))
 165             return u"<a href={}>{}</a>".format(
 166                 make_link(dl_dict['File'], dl_dict['Line']), value)
 167         else:
 168             return value
 169
 170     # Return a cached dictionary for the arguments.  The key for each entry is
 171     # the argument key (e.g. 'Callee' for inlining remarks.  The value is a
 172     # list containing the value (e.g. for 'Callee' the function) and
 173     # optionally a DebugLoc.
 174     def getArgDict(self):
 175         if hasattr(self, 'ArgDict'):
 176             return self.ArgDict
 177         self.ArgDict = {}
 178         for arg in self.Args:
 179             if len(arg) == 2:
 180                 if arg[0][0] == 'DebugLoc':
 181                     dbgidx = 0
 182                 else:
 183                     assert(arg[1][0] == 'DebugLoc')
 184                     dbgidx = 1
 185
 186                 key = arg[1 - dbgidx][0]
 187                 entry = (arg[1 - dbgidx][1], arg[dbgidx][1])
 188             else:
 189                 arg = arg[0]
 190                 key = arg[0]
 191                 entry = (arg[1], )
 192
 193             self.ArgDict[key] = entry
 194         return self.ArgDict
 195
 196     def getDiffPrefix(self):
 197         if hasattr(self, 'Added'):
 198             if self.Added:
 199                 return '+'
 200             else:
 201                 return '-'
 202         return ''
 203
 204     @property
 205     def PassWithDiffPrefix(self):
 206         return self.getDiffPrefix() + self.Pass
 207
 208     @property
 209     def message(self):
 210         # Args is a list of mappings (dictionaries)
 211         values = [self.getArgString(mapping) for mapping in self.Args]
 212         return "".join(values)
 213
 214     @property
 215     def RelativeHotness(self):
 216         if self.max_hotness:
 217             return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness)
 218         else:
 219             return ''
 220
 221     @property
 222     def key(self):
 223         return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
 224                 self.Line, self.Column, self.Function, self.Args)
 225
 226     def __hash__(self):
 227         return hash(self.key)
 228
 229     def __eq__(self, other):
 230         return self.key == other.key
 231
 232     def __repr__(self):
 233         return str(self.key)
 234
 235
 236 class Analysis(Remark):
 237     yaml_tag = '!Analysis'
 238
 239     @property
 240     def color(self):
 241         return "white"
 242
 243
 244 class AnalysisFPCommute(Analysis):
 245     yaml_tag = '!AnalysisFPCommute'
 246
 247
 248 class AnalysisAliasing(Analysis):
 249     yaml_tag = '!AnalysisAliasing'
 250
 251
 252 class Passed(Remark):
 253     yaml_tag = '!Passed'
 254
 255     @property
 256     def color(self):
 257         return "green"
 258
 259
 260 class Missed(Remark):
 261     yaml_tag = '!Missed'
 262
 263     @property
 264     def color(self):
 265         return "red"
 266
 267
 268 def get_remarks(input_file, filter_):
 269     max_hotness = 0
 270     all_remarks = dict()
 271     file_remarks = defaultdict(functools.partial(defaultdict, list))
 272
 273     with open(input_file) as f:
 274         docs = yaml.load_all(f, Loader=Loader)
 275
 276         filter_e = re.compile(filter_)
 277         for remark in docs:
 278             remark.canonicalize()
 279             # Avoid remarks withoug debug location or if they are duplicated
 280             if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
 281                 continue
 282
 283             if filter_ and not filter_e.search(remark.Pass):
 284                 continue
 285
 286             all_remarks[remark.key] = remark
 287
 288             file_remarks[remark.File][remark.Line].append(remark)
 289
 290             # If we're reading a back a diff yaml file, max_hotness is already
 291             # captured which may actually be less than the max hotness found
 292             # in the file.
 293             if hasattr(remark, 'max_hotness'):
 294                 max_hotness = remark.max_hotness
 295             max_hotness = max(max_hotness, remark.Hotness)
 296
 297     return max_hotness, all_remarks, file_remarks
 298
 299
 300 def gather_results(filenames, num_jobs, should_print_progress, filter_):
 301     if should_print_progress:
 302         print('Reading YAML files...')
 303     if not Remark.demangler_proc:
 304         Remark.set_demangler(Remark.default_demangler)
 305     remarks = optpmap.pmap(
 306         get_remarks, filenames, num_jobs, should_print_progress, filter_)
 307     max_hotness = max(entry[0] for entry in remarks)
 308
 309     def merge_file_remarks(file_remarks_job, all_remarks, merged):
 310         for filename, d in iteritems(file_remarks_job):
 311             for line, remarks in iteritems(d):
 312                 for remark in remarks:
 313                     # Bring max_hotness into the remarks so that
 314                     # RelativeHotness does not depend on an external global.
 315                     remark.max_hotness = max_hotness
 316                     if remark.key not in all_remarks:
 317                         merged[filename][line].append(remark)
 318
 319     all_remarks = dict()
 320     file_remarks = defaultdict(functools.partial(defaultdict, list))
 321     for _, all_remarks_job, file_remarks_job in remarks:
 322         merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
 323         all_remarks.update(all_remarks_job)
 324
 325     return all_remarks, file_remarks, max_hotness != 0
 326
 327
 328 def find_opt_files(*dirs_or_files):
 329     all = []
 330     for dir_or_file in dirs_or_files:
 331         if os.path.isfile(dir_or_file):
 332             all.append(dir_or_file)
 333         else:
 334             for dir, subdirs, files in os.walk(dir_or_file):
 335                 # Exclude mounted directories and symlinks (os.walk default).
 336                 subdirs[:] = [d for d in subdirs
 337                               if not os.path.ismount(os.path.join(dir, d))]
 338                 for file in files:
 339                     if fnmatch.fnmatch(file, "*.opt.yaml*"):
 340                         all.append(os.path.join(dir, file))
 341     return all