tools/opt-viewer/opt-viewer.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import argparse
   6 import cgi
   7 import codecs
   8 import errno
   9 import functools
  10 from multiprocessing import cpu_count
  11 import os.path
  12 import re
  13 import shutil
  14 import sys
  15
  16 from pygments import highlight
  17 from pygments.lexers.c_cpp import CppLexer
  18 from pygments.formatters import HtmlFormatter
  19
  20 import optpmap
  21 import optrecord
  22
  23
  24 desc = '''Generate HTML output to visualize optimization records from the YAML files
  25 generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
  26
  27 The tools requires PyYAML and Pygments Python packages.'''
  28
  29
  30 # This allows passing the global context to the child processes.
  31 class Context:
  32     def __init__(self, caller_loc = dict()):
  33        # Map function names to their source location for function where inlining happened
  34        self.caller_loc = caller_loc
  35
  36 context = Context()
  37
  38 def suppress(remark):
  39     if remark.Name == 'sil.Specialized':
  40         return remark.getArgDict()['Function'][0].startswith('\"Swift.')
  41     elif remark.Name == 'sil.Inlined':
  42         return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
  43     return False
  44
  45 class SourceFileRenderer:
  46     def __init__(self, source_dir, output_dir, filename, no_highlight):
  47         self.filename = filename
  48         existing_filename = None
  49         if os.path.exists(filename):
  50             existing_filename = filename
  51         else:
  52             fn = os.path.join(source_dir, filename)
  53             if os.path.exists(fn):
  54                 existing_filename = fn
  55
  56         self.no_highlight = no_highlight
  57         self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
  58         if existing_filename:
  59             self.source_stream = open(existing_filename)
  60         else:
  61             self.source_stream = None
  62             print('''
  63 <html>
  64 <h1>Unable to locate file {}</h1>
  65 </html>
  66             '''.format(filename), file=self.stream)
  67
  68         self.html_formatter = HtmlFormatter(encoding='utf-8')
  69         self.cpp_lexer = CppLexer(stripnl=False)
  70
  71     def render_source_lines(self, stream, line_remarks):
  72         file_text = stream.read()
  73
  74         if self.no_highlight:
  75             if sys.version_info.major >= 3:
  76                 html_highlighted = file_text
  77             else:
  78                 html_highlighted = file_text.decode('utf-8')
  79         else:
  80             html_highlighted = highlight(
  81             file_text,
  82                 self.cpp_lexer,
  83                 self.html_formatter)
  84
  85             # Note that the API is different between Python 2 and 3.  On
  86             # Python 3, pygments.highlight() returns a bytes object, so we
  87             # have to decode.  On Python 2, the output is str but since we
  88             # support unicode characters and the output streams is unicode we
  89             # decode too.
  90             html_highlighted = html_highlighted.decode('utf-8')
  91
  92             # Take off the header and footer, these must be
  93             #   reapplied line-wise, within the page structure
  94             html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
  95             html_highlighted = html_highlighted.replace('</pre></div>', '')
  96
  97         for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
  98             print(u'''
  99 <tr>
 100 <td><a name=\"L{linenum}\">{linenum}</a></td>
 101 <td></td>
 102 <td></td>
 103 <td><div class="highlight"><pre>{html_line}</pre></div></td>
 104 </tr>'''.format(**locals()), file=self.stream)
 105
 106             for remark in line_remarks.get(linenum, []):
 107                 if not suppress(remark):
 108                     self.render_inline_remarks(remark, html_line)
 109
 110     def render_inline_remarks(self, r, line):
 111         inlining_context = r.DemangledFunctionName
 112         dl = context.caller_loc.get(r.Function)
 113         if dl:
 114             dl_dict = dict(list(dl))
 115             link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
 116             inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
 117
 118         # Column is the number of characters *including* tabs, keep those and
 119         # replace everything else with spaces.
 120         indent = line[:max(r.Column, 1) - 1]
 121         indent = re.sub('\S', ' ', indent)
 122
 123         # Create expanded message and link if we have a multiline message.
 124         lines = r.message.split('\n')
 125         if len(lines) > 1:
 126             expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
 127             message = lines[0]
 128             expand_message = u'''
 129 <div class="full-info" style="display:none;">
 130   <div class="col-left"><pre style="display:inline">{}</pre></div>
 131   <div class="expanded col-left"><pre>{}</pre></div>
 132 </div>'''.format(indent, '\n'.join(lines[1:]))
 133         else:
 134             expand_link = ''
 135             expand_message = ''
 136             message = r.message
 137         print(u'''
 138 <tr>
 139 <td></td>
 140 <td>{r.RelativeHotness}</td>
 141 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 142 <td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message}&nbsp;</span>{expand_message}</td>
 143 <td class=\"column-entry-yellow\">{inlining_context}</td>
 144 </tr>'''.format(**locals()), file=self.stream)
 145
 146     def render(self, line_remarks):
 147         if not self.source_stream:
 148             return
 149
 150         print('''
 151 <html>
 152 <title>{}</title>
 153 <meta charset="utf-8" />
 154 <head>
 155 <link rel='stylesheet' type='text/css' href='style.css'>
 156 <script type="text/javascript">
 157 /* Simple helper to show/hide the expanded message of a remark. */
 158 function toggleExpandedMessage(e) {{
 159   var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
 160   if (!FullTextElems || FullTextElems.length < 1) {{
 161       return false;
 162   }}
 163   var FullText = FullTextElems[0];
 164   if (FullText.style.display == 'none') {{
 165     e.innerHTML = '-';
 166     FullText.style.display = 'block';
 167   }} else {{
 168     e.innerHTML = '+';
 169     FullText.style.display = 'none';
 170   }}
 171 }}
 172 </script>
 173 </head>
 174 <body>
 175 <div class="centered">
 176 <table class="source">
 177 <thead>
 178 <tr>
 179 <th style="width: 2%">Line</td>
 180 <th style="width: 3%">Hotness</td>
 181 <th style="width: 10%">Optimization</td>
 182 <th style="width: 70%">Source</td>
 183 <th style="width: 15%">Inline Context</td>
 184 </tr>
 185 </thead>
 186 <tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
 187         self.render_source_lines(self.source_stream, line_remarks)
 188
 189         print('''
 190 </tbody>
 191 </table>
 192 </body>
 193 </html>''', file=self.stream)
 194
 195
 196 class IndexRenderer:
 197     def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
 198         self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
 199         self.should_display_hotness = should_display_hotness
 200         self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
 201
 202     def render_entry(self, r, odd):
 203         escaped_name = cgi.escape(r.DemangledFunctionName)
 204         print(u'''
 205 <tr>
 206 <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
 207 <td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
 208 <td class=\"column-entry-{odd}\">{escaped_name}</td>
 209 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 210 </tr>'''.format(**locals()), file=self.stream)
 211
 212     def render(self, all_remarks):
 213         print('''
 214 <html>
 215 <meta charset="utf-8" />
 216 <head>
 217 <link rel='stylesheet' type='text/css' href='style.css'>
 218 </head>
 219 <body>
 220 <div class="centered">
 221 <table>
 222 <tr>
 223 <td>Source Location</td>
 224 <td>Hotness</td>
 225 <td>Function</td>
 226 <td>Pass</td>
 227 </tr>''', file=self.stream)
 228
 229         max_entries = None
 230         if self.should_display_hotness:
 231             max_entries = self.max_hottest_remarks_on_index
 232
 233         for i, remark in enumerate(all_remarks[:max_entries]):
 234             if not suppress(remark):
 235                 self.render_entry(remark, i % 2)
 236         print('''
 237 </table>
 238 </body>
 239 </html>''', file=self.stream)
 240
 241
 242 def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
 243     global context
 244     context = ctx
 245     filename, remarks = entry
 246     SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
 247
 248
 249 def map_remarks(all_remarks):
 250     # Set up a map between function names and their source location for
 251     # function where inlining happened
 252     for remark in optrecord.itervalues(all_remarks):
 253         if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
 254             for arg in remark.Args:
 255                 arg_dict = dict(list(arg))
 256                 caller = arg_dict.get('Caller')
 257                 if caller:
 258                     try:
 259                         context.caller_loc[caller] = arg_dict['DebugLoc']
 260                     except KeyError:
 261                         pass
 262
 263
 264 def generate_report(all_remarks,
 265                     file_remarks,
 266                     source_dir,
 267                     output_dir,
 268                     no_highlight,
 269                     should_display_hotness,
 270                     max_hottest_remarks_on_index,
 271                     num_jobs,
 272                     should_print_progress):
 273     try:
 274         os.makedirs(output_dir)
 275     except OSError as e:
 276         if e.errno == errno.EEXIST and os.path.isdir(output_dir):
 277             pass
 278         else:
 279             raise
 280
 281     if should_print_progress:
 282         print('Rendering index page...')
 283     if should_display_hotness:
 284         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
 285     else:
 286         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
 287     IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
 288
 289     shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
 290             "style.css"), output_dir)
 291
 292     _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
 293     if should_print_progress:
 294         print('Rendering HTML files...')
 295     optpmap.pmap(_render_file_bound,
 296                  file_remarks.items(),
 297                  num_jobs,
 298                  should_print_progress)
 299
 300
 301 def main():
 302     parser = argparse.ArgumentParser(description=desc)
 303     parser.add_argument(
 304         'yaml_dirs_or_files',
 305         nargs='+',
 306         help='List of optimization record files or directories searched '
 307              'for optimization record files.')
 308     parser.add_argument(
 309         '--output-dir',
 310         '-o',
 311         default='html',
 312         help='Path to a directory where generated HTML files will be output. '
 313              'If the directory does not already exist, it will be created. '
 314              '"%(default)s" by default.')
 315     parser.add_argument(
 316         '--jobs',
 317         '-j',
 318         default=None,
 319         type=int,
 320         help='Max job count (defaults to %(default)s, the current CPU count)')
 321     parser.add_argument(
 322         '--source-dir',
 323         '-s',
 324         default='',
 325         help='set source directory')
 326     parser.add_argument(
 327         '--no-progress-indicator',
 328         '-n',
 329         action='store_true',
 330         default=False,
 331         help='Do not display any indicator of how many YAML files were read '
 332              'or rendered into HTML.')
 333     parser.add_argument(
 334         '--max-hottest-remarks-on-index',
 335         default=1000,
 336         type=int,
 337         help='Maximum number of the hottest remarks to appear on the index page')
 338     parser.add_argument(
 339         '--no-highlight',
 340         action='store_true',
 341         default=False,
 342         help='Do not use a syntax highlighter when rendering the source code')
 343     parser.add_argument(
 344         '--demangler',
 345         help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
 346
 347     parser.add_argument(
 348         '--filter',
 349         default='',
 350         help='Only display remarks from passes matching filter expression')
 351
 352     # Do not make this a global variable.  Values needed to be propagated through
 353     # to individual classes and functions to be portable with multiprocessing across
 354     # Windows and non-Windows.
 355     args = parser.parse_args()
 356
 357     print_progress = not args.no_progress_indicator
 358     if args.demangler:
 359         optrecord.Remark.set_demangler(args.demangler)
 360
 361     files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
 362     if not files:
 363         parser.error("No *.opt.yaml files found")
 364         sys.exit(1)
 365
 366     all_remarks, file_remarks, should_display_hotness = \
 367         optrecord.gather_results(files, args.jobs, print_progress, args.filter)
 368
 369     map_remarks(all_remarks)
 370
 371     generate_report(all_remarks,
 372                     file_remarks,
 373                     args.source_dir,
 374                     args.output_dir,
 375                     args.no_highlight,
 376                     should_display_hotness,
 377                     args.max_hottest_remarks_on_index,
 378                     args.jobs,
 379                     print_progress)
 380
 381 if __name__ == '__main__':
 382     main()