llvm/tools/opt-viewer/optrecord.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import io
   6 import yaml
   7 # Try to use the C parser.
   8 try:
   9     from yaml import CLoader as Loader
  10 except ImportError:
  11     print("For faster parsing, you may want to install libYAML for PyYAML")
  12     from yaml import Loader
  13
  14 import html
  15 from collections import defaultdict
  16 import fnmatch
  17 import functools
  18 from multiprocessing import Lock
  19 import os, os.path
  20 import subprocess
  21 try:
  22     # The previously builtin function `intern()` was moved
  23     # to the `sys` module in Python 3.
  24     from sys import intern
  25 except:
  26     pass
  27
  28 import re
  29
  30 import optpmap
  31
  32 try:
  33     dict.iteritems
  34 except AttributeError:
  35     # Python 3
  36     def itervalues(d):
  37         return iter(d.values())
  38     def iteritems(d):
  39         return iter(d.items())
  40 else:
  41     # Python 2
  42     def itervalues(d):
  43         return d.itervalues()
  44     def iteritems(d):
  45         return d.iteritems()
  46
  47
  48 def html_file_name(filename):
  49     return filename.replace('/', '_').replace('#', '_') + ".html"
  50
  51
  52 def make_link(File, Line):
  53     return "\"{}#L{}\"".format(html_file_name(File), Line)
  54
  55
  56 class Remark(yaml.YAMLObject):
  57     # Work-around for http://pyyaml.org/ticket/154.
  58     yaml_loader = Loader
  59
  60     default_demangler = 'c++filt -n'
  61     demangler_proc = None
  62
  63     @classmethod
  64     def set_demangler(cls, demangler):
  65         cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  66         cls.demangler_lock = Lock()
  67
  68     @classmethod
  69     def demangle(cls, name):
  70         with cls.demangler_lock:
  71             cls.demangler_proc.stdin.write((name + '\n').encode('utf-8'))
  72             cls.demangler_proc.stdin.flush()
  73             return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8')
  74
  75     # Intern all strings since we have lot of duplication across filenames,
  76     # remark text.
  77     #
  78     # Change Args from a list of dicts to a tuple of tuples.  This saves
  79     # memory in two ways.  One, a small tuple is significantly smaller than a
  80     # small dict.  Two, using tuple instead of list allows Args to be directly
  81     # used as part of the key (in Python only immutable types are hashable).
  82     def _reduce_memory(self):
  83         self.Pass = intern(self.Pass)
  84         self.Name = intern(self.Name)
  85         try:
  86             # Can't intern unicode strings.
  87             self.Function = intern(self.Function)
  88         except:
  89             pass
  90
  91         def _reduce_memory_dict(old_dict):
  92             new_dict = dict()
  93             for (k, v) in iteritems(old_dict):
  94                 if type(k) is str:
  95                     k = intern(k)
  96
  97                 if type(v) is str:
  98                     v = intern(v)
  99                 elif type(v) is dict:
 100                     # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
 101                     v = _reduce_memory_dict(v)
 102                 new_dict[k] = v
 103             return tuple(new_dict.items())
 104
 105         self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
 106
 107     # The inverse operation of the dictonary-related memory optimization in
 108     # _reduce_memory_dict.  E.g.
 109     #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
 110     def recover_yaml_structure(self):
 111         def tuple_to_dict(t):
 112             d = dict()
 113             for (k, v) in t:
 114                 if type(v) is tuple:
 115                     v = tuple_to_dict(v)
 116                 d[k] = v
 117             return d
 118
 119         self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
 120
 121     def canonicalize(self):
 122         if not hasattr(self, 'Hotness'):
 123             self.Hotness = 0
 124         if not hasattr(self, 'Args'):
 125             self.Args = []
 126         self._reduce_memory()
 127
 128     @property
 129     def File(self):
 130         return self.DebugLoc['File']
 131
 132     @property
 133     def Line(self):
 134         return int(self.DebugLoc['Line'])
 135
 136     @property
 137     def Column(self):
 138         return self.DebugLoc['Column']
 139
 140     @property
 141     def DebugLocString(self):
 142         return "{}:{}:{}".format(self.File, self.Line, self.Column)
 143
 144     @property
 145     def DemangledFunctionName(self):
 146         return self.demangle(self.Function)
 147
 148     @property
 149     def Link(self):
 150         return make_link(self.File, self.Line)
 151
 152     def getArgString(self, mapping):
 153         mapping = dict(list(mapping))
 154         dl = mapping.get('DebugLoc')
 155         if dl:
 156             del mapping['DebugLoc']
 157
 158         assert(len(mapping) == 1)
 159         (key, value) = list(mapping.items())[0]
 160
 161         if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
 162             value = html.escape(self.demangle(value))
 163
 164         if dl and key != 'Caller':
 165             dl_dict = dict(list(dl))
 166             return u"<a href={}>{}</a>".format(
 167                 make_link(dl_dict['File'], dl_dict['Line']), value)
 168         else:
 169             return value
 170
 171     # Return a cached dictionary for the arguments.  The key for each entry is
 172     # the argument key (e.g. 'Callee' for inlining remarks.  The value is a
 173     # list containing the value (e.g. for 'Callee' the function) and
 174     # optionally a DebugLoc.
 175     def getArgDict(self):
 176         if hasattr(self, 'ArgDict'):
 177             return self.ArgDict
 178         self.ArgDict = {}
 179         for arg in self.Args:
 180             if len(arg) == 2:
 181                 if arg[0][0] == 'DebugLoc':
 182                     dbgidx = 0
 183                 else:
 184                     assert(arg[1][0] == 'DebugLoc')
 185                     dbgidx = 1
 186
 187                 key = arg[1 - dbgidx][0]
 188                 entry = (arg[1 - dbgidx][1], arg[dbgidx][1])
 189             else:
 190                 arg = arg[0]
 191                 key = arg[0]
 192                 entry = (arg[1], )
 193
 194             self.ArgDict[key] = entry
 195         return self.ArgDict
 196
 197     def getDiffPrefix(self):
 198         if hasattr(self, 'Added'):
 199             if self.Added:
 200                 return '+'
 201             else:
 202                 return '-'
 203         return ''
 204
 205     @property
 206     def PassWithDiffPrefix(self):
 207         return self.getDiffPrefix() + self.Pass
 208
 209     @property
 210     def message(self):
 211         # Args is a list of mappings (dictionaries)
 212         values = [self.getArgString(mapping) for mapping in self.Args]
 213         return "".join(values)
 214
 215     @property
 216     def RelativeHotness(self):
 217         if self.max_hotness:
 218             return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness)
 219         else:
 220             return ''
 221
 222     @property
 223     def key(self):
 224         return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
 225                 self.Line, self.Column, self.Function, self.Args)
 226
 227     def __hash__(self):
 228         return hash(self.key)
 229
 230     def __eq__(self, other):
 231         return self.key == other.key
 232
 233     def __repr__(self):
 234         return str(self.key)
 235
 236
 237 class Analysis(Remark):
 238     yaml_tag = '!Analysis'
 239
 240     @property
 241     def color(self):
 242         return "white"
 243
 244
 245 class AnalysisFPCommute(Analysis):
 246     yaml_tag = '!AnalysisFPCommute'
 247
 248
 249 class AnalysisAliasing(Analysis):
 250     yaml_tag = '!AnalysisAliasing'
 251
 252
 253 class Passed(Remark):
 254     yaml_tag = '!Passed'
 255
 256     @property
 257     def color(self):
 258         return "green"
 259
 260
 261 class Missed(Remark):
 262     yaml_tag = '!Missed'
 263
 264     @property
 265     def color(self):
 266         return "red"
 267
 268 class Failure(Missed):
 269     yaml_tag = '!Failure'
 270
 271 def get_remarks(input_file, filter_=None):
 272     max_hotness = 0
 273     all_remarks = dict()
 274     file_remarks = defaultdict(functools.partial(defaultdict, list))
 275
 276     with io.open(input_file, encoding = 'utf-8') as f:
 277         docs = yaml.load_all(f, Loader=Loader)
 278
 279         filter_e = None
 280         if filter_:
 281             filter_e = re.compile(filter_)
 282         for remark in docs:
 283             remark.canonicalize()
 284             # Avoid remarks withoug debug location or if they are duplicated
 285             if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
 286                 continue
 287
 288             if filter_e and not filter_e.search(remark.Pass):
 289                 continue
 290
 291             all_remarks[remark.key] = remark
 292
 293             file_remarks[remark.File][remark.Line].append(remark)
 294
 295             # If we're reading a back a diff yaml file, max_hotness is already
 296             # captured which may actually be less than the max hotness found
 297             # in the file.
 298             if hasattr(remark, 'max_hotness'):
 299                 max_hotness = remark.max_hotness
 300             max_hotness = max(max_hotness, remark.Hotness)
 301
 302     return max_hotness, all_remarks, file_remarks
 303
 304
 305 def gather_results(filenames, num_jobs, should_print_progress, filter_=None):
 306     if should_print_progress:
 307         print('Reading YAML files...')
 308     if not Remark.demangler_proc:
 309         Remark.set_demangler(Remark.default_demangler)
 310     remarks = optpmap.pmap(
 311         get_remarks, filenames, num_jobs, should_print_progress, filter_)
 312     max_hotness = max(entry[0] for entry in remarks)
 313
 314     def merge_file_remarks(file_remarks_job, all_remarks, merged):
 315         for filename, d in iteritems(file_remarks_job):
 316             for line, remarks in iteritems(d):
 317                 for remark in remarks:
 318                     # Bring max_hotness into the remarks so that
 319                     # RelativeHotness does not depend on an external global.
 320                     remark.max_hotness = max_hotness
 321                     if remark.key not in all_remarks:
 322                         merged[filename][line].append(remark)
 323
 324     all_remarks = dict()
 325     file_remarks = defaultdict(functools.partial(defaultdict, list))
 326     for _, all_remarks_job, file_remarks_job in remarks:
 327         merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
 328         all_remarks.update(all_remarks_job)
 329
 330     return all_remarks, file_remarks, max_hotness != 0
 331
 332
 333 def find_opt_files(*dirs_or_files):
 334     all = []
 335     for dir_or_file in dirs_or_files:
 336         if os.path.isfile(dir_or_file):
 337             all.append(dir_or_file)
 338         else:
 339             for dir, subdirs, files in os.walk(dir_or_file):
 340                 # Exclude mounted directories and symlinks (os.walk default).
 341                 subdirs[:] = [d for d in subdirs
 342                               if not os.path.ismount(os.path.join(dir, d))]
 343                 for file in files:
 344                     if fnmatch.fnmatch(file, "*.opt.yaml*"):
 345                         all.append(os.path.join(dir, file))
 346     return all