tools/opt-viewer/optrecord.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import yaml
   6 # Try to use the C parser.
   7 try:
   8     from yaml import CLoader as Loader
   9 except ImportError:
  10     print("For faster parsing, you may want to install libYAML for PyYAML")
  11     from yaml import Loader
  12
  13 import cgi
  14 from collections import defaultdict
  15 import fnmatch
  16 import functools
  17 from multiprocessing import Lock
  18 import os, os.path
  19 import subprocess
  20 try:
  21     # The previously builtin function `intern()` was moved
  22     # to the `sys` module in Python 3.
  23     from sys import intern
  24 except:
  25     pass
  26
  27 import re
  28
  29 import optpmap
  30
  31 try:
  32     dict.iteritems
  33 except AttributeError:
  34     # Python 3
  35     def itervalues(d):
  36         return iter(d.values())
  37     def iteritems(d):
  38         return iter(d.items())
  39 else:
  40     # Python 2
  41     def itervalues(d):
  42         return d.itervalues()
  43     def iteritems(d):
  44         return d.iteritems()
  45
  46
  47 def html_file_name(filename):
  48     return filename.replace('/', '_').replace('#', '_') + ".html"
  49
  50
  51 def make_link(File, Line):
  52     return "\"{}#L{}\"".format(html_file_name(File), Line)
  53
  54
  55 class Remark(yaml.YAMLObject):
  56     # Work-around for http://pyyaml.org/ticket/154.
  57     yaml_loader = Loader
  58
  59     default_demangler = 'c++filt -n'
  60     demangler_proc = None
  61
  62     @classmethod
  63     def set_demangler(cls, demangler):
  64         cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  65         cls.demangler_lock = Lock()
  66
  67     @classmethod
  68     def demangle(cls, name):
  69         with cls.demangler_lock:
  70             cls.demangler_proc.stdin.write((name + '\n').encode('utf-8'))
  71             cls.demangler_proc.stdin.flush()
  72             return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8')
  73
  74     # Intern all strings since we have lot of duplication across filenames,
  75     # remark text.
  76     #
  77     # Change Args from a list of dicts to a tuple of tuples.  This saves
  78     # memory in two ways.  One, a small tuple is significantly smaller than a
  79     # small dict.  Two, using tuple instead of list allows Args to be directly
  80     # used as part of the key (in Python only immutable types are hashable).
  81     def _reduce_memory(self):
  82         self.Pass = intern(self.Pass)
  83         self.Name = intern(self.Name)
  84         try:
  85             # Can't intern unicode strings.
  86             self.Function = intern(self.Function)
  87         except:
  88             pass
  89
  90         def _reduce_memory_dict(old_dict):
  91             new_dict = dict()
  92             for (k, v) in iteritems(old_dict):
  93                 if type(k) is str:
  94                     k = intern(k)
  95
  96                 if type(v) is str:
  97                     v = intern(v)
  98                 elif type(v) is dict:
  99                     # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
 100                     v = _reduce_memory_dict(v)
 101                 new_dict[k] = v
 102             return tuple(new_dict.items())
 103
 104         self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
 105
 106     # The inverse operation of the dictonary-related memory optimization in
 107     # _reduce_memory_dict.  E.g.
 108     #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
 109     def recover_yaml_structure(self):
 110         def tuple_to_dict(t):
 111             d = dict()
 112             for (k, v) in t:
 113                 if type(v) is tuple:
 114                     v = tuple_to_dict(v)
 115                 d[k] = v
 116             return d
 117
 118         self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
 119
 120     def canonicalize(self):
 121         if not hasattr(self, 'Hotness'):
 122             self.Hotness = 0
 123         if not hasattr(self, 'Args'):
 124             self.Args = []
 125         self._reduce_memory()
 126
 127     @property
 128     def File(self):
 129         return self.DebugLoc['File']
 130
 131     @property
 132     def Line(self):
 133         return int(self.DebugLoc['Line'])
 134
 135     @property
 136     def Column(self):
 137         return self.DebugLoc['Column']
 138
 139     @property
 140     def DebugLocString(self):
 141         return "{}:{}:{}".format(self.File, self.Line, self.Column)
 142
 143     @property
 144     def DemangledFunctionName(self):
 145         return self.demangle(self.Function)
 146
 147     @property
 148     def Link(self):
 149         return make_link(self.File, self.Line)
 150
 151     def getArgString(self, mapping):
 152         mapping = dict(list(mapping))
 153         dl = mapping.get('DebugLoc')
 154         if dl:
 155             del mapping['DebugLoc']
 156
 157         assert(len(mapping) == 1)
 158         (key, value) = list(mapping.items())[0]
 159
 160         if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
 161             value = cgi.escape(self.demangle(value))
 162
 163         if dl and key != 'Caller':
 164             dl_dict = dict(list(dl))
 165             return u"<a href={}>{}</a>".format(
 166                 make_link(dl_dict['File'], dl_dict['Line']), value)
 167         else:
 168             return value
 169
 170     # Return a cached dictionary for the arguments.  The key for each entry is
 171     # the argument key (e.g. 'Callee' for inlining remarks.  The value is a
 172     # list containing the value (e.g. for 'Callee' the function) and
 173     # optionally a DebugLoc.
 174     def getArgDict(self):
 175         if hasattr(self, 'ArgDict'):
 176             return self.ArgDict
 177         self.ArgDict = {}
 178         for arg in self.Args:
 179             if len(arg) == 2:
 180                 if arg[0][0] == 'DebugLoc':
 181                     dbgidx = 0
 182                 else:
 183                     assert(arg[1][0] == 'DebugLoc')
 184                     dbgidx = 1
 185
 186                 key = arg[1 - dbgidx][0]
 187                 entry = (arg[1 - dbgidx][1], arg[dbgidx][1])
 188             else:
 189                 arg = arg[0]
 190                 key = arg[0]
 191                 entry = (arg[1], )
 192
 193             self.ArgDict[key] = entry
 194         return self.ArgDict
 195
 196     def getDiffPrefix(self):
 197         if hasattr(self, 'Added'):
 198             if self.Added:
 199                 return '+'
 200             else:
 201                 return '-'
 202         return ''
 203
 204     @property
 205     def PassWithDiffPrefix(self):
 206         return self.getDiffPrefix() + self.Pass
 207
 208     @property
 209     def message(self):
 210         # Args is a list of mappings (dictionaries)
 211         values = [self.getArgString(mapping) for mapping in self.Args]
 212         return "".join(values)
 213
 214     @property
 215     def RelativeHotness(self):
 216         if self.max_hotness:
 217             return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness)
 218         else:
 219             return ''
 220
 221     @property
 222     def key(self):
 223         return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
 224                 self.Line, self.Column, self.Function, self.Args)
 225
 226     def __hash__(self):
 227         return hash(self.key)
 228
 229     def __eq__(self, other):
 230         return self.key == other.key
 231
 232     def __repr__(self):
 233         return str(self.key)
 234
 235
 236 class Analysis(Remark):
 237     yaml_tag = '!Analysis'
 238
 239     @property
 240     def color(self):
 241         return "white"
 242
 243
 244 class AnalysisFPCommute(Analysis):
 245     yaml_tag = '!AnalysisFPCommute'
 246
 247
 248 class AnalysisAliasing(Analysis):
 249     yaml_tag = '!AnalysisAliasing'
 250
 251
 252 class Passed(Remark):
 253     yaml_tag = '!Passed'
 254
 255     @property
 256     def color(self):
 257         return "green"
 258
 259
 260 class Missed(Remark):
 261     yaml_tag = '!Missed'
 262
 263     @property
 264     def color(self):
 265         return "red"
 266
 267 class Failure(Missed):
 268     yaml_tag = '!Failure'
 269
 270 def get_remarks(input_file, filter_=None):
 271     max_hotness = 0
 272     all_remarks = dict()
 273     file_remarks = defaultdict(functools.partial(defaultdict, list))
 274
 275     with open(input_file) as f:
 276         docs = yaml.load_all(f, Loader=Loader)
 277
 278         filter_e = None
 279         if filter_:
 280             filter_e = re.compile(filter_)
 281         for remark in docs:
 282             remark.canonicalize()
 283             # Avoid remarks withoug debug location or if they are duplicated
 284             if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
 285                 continue
 286
 287             if filter_e and not filter_e.search(remark.Pass):
 288                 continue
 289
 290             all_remarks[remark.key] = remark
 291
 292             file_remarks[remark.File][remark.Line].append(remark)
 293
 294             # If we're reading a back a diff yaml file, max_hotness is already
 295             # captured which may actually be less than the max hotness found
 296             # in the file.
 297             if hasattr(remark, 'max_hotness'):
 298                 max_hotness = remark.max_hotness
 299             max_hotness = max(max_hotness, remark.Hotness)
 300
 301     return max_hotness, all_remarks, file_remarks
 302
 303
 304 def gather_results(filenames, num_jobs, should_print_progress, filter_=None):
 305     if should_print_progress:
 306         print('Reading YAML files...')
 307     if not Remark.demangler_proc:
 308         Remark.set_demangler(Remark.default_demangler)
 309     remarks = optpmap.pmap(
 310         get_remarks, filenames, num_jobs, should_print_progress, filter_)
 311     max_hotness = max(entry[0] for entry in remarks)
 312
 313     def merge_file_remarks(file_remarks_job, all_remarks, merged):
 314         for filename, d in iteritems(file_remarks_job):
 315             for line, remarks in iteritems(d):
 316                 for remark in remarks:
 317                     # Bring max_hotness into the remarks so that
 318                     # RelativeHotness does not depend on an external global.
 319                     remark.max_hotness = max_hotness
 320                     if remark.key not in all_remarks:
 321                         merged[filename][line].append(remark)
 322
 323     all_remarks = dict()
 324     file_remarks = defaultdict(functools.partial(defaultdict, list))
 325     for _, all_remarks_job, file_remarks_job in remarks:
 326         merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
 327         all_remarks.update(all_remarks_job)
 328
 329     return all_remarks, file_remarks, max_hotness != 0
 330
 331
 332 def find_opt_files(*dirs_or_files):
 333     all = []
 334     for dir_or_file in dirs_or_files:
 335         if os.path.isfile(dir_or_file):
 336             all.append(dir_or_file)
 337         else:
 338             for dir, subdirs, files in os.walk(dir_or_file):
 339                 # Exclude mounted directories and symlinks (os.walk default).
 340                 subdirs[:] = [d for d in subdirs
 341                               if not os.path.ismount(os.path.join(dir, d))]
 342                 for file in files:
 343                     if fnmatch.fnmatch(file, "*.opt.yaml*"):
 344                         all.append(os.path.join(dir, file))
 345     return all