llvm/tools/opt-viewer/opt-viewer.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import argparse
   6 import errno
   7 import functools
   8 import html
   9 import io
  10 from multiprocessing import cpu_count
  11 import os.path
  12 import re
  13 import shutil
  14 import sys
  15
  16 from pygments import highlight
  17 from pygments.lexers.c_cpp import CppLexer
  18 from pygments.formatters import HtmlFormatter
  19
  20 import optpmap
  21 import optrecord
  22
  23
  24 desc = '''Generate HTML output to visualize optimization records from the YAML files
  25 generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
  26
  27 The tools requires PyYAML and Pygments Python packages.'''
  28
  29
  30 # This allows passing the global context to the child processes.
  31 class Context:
  32     def __init__(self, caller_loc = dict()):
  33        # Map function names to their source location for function where inlining happened
  34        self.caller_loc = caller_loc
  35
  36 context = Context()
  37
  38 def suppress(remark):
  39     if remark.Name == 'sil.Specialized':
  40         return remark.getArgDict()['Function'][0].startswith('\"Swift.')
  41     elif remark.Name == 'sil.Inlined':
  42         return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
  43     return False
  44
  45 class SourceFileRenderer:
  46     def __init__(self, source_dir, output_dir, filename, no_highlight):
  47         self.filename = filename
  48         existing_filename = None
  49         if os.path.exists(filename):
  50             existing_filename = filename
  51         else:
  52             fn = os.path.join(source_dir, filename)
  53             if os.path.exists(fn):
  54                 existing_filename = fn
  55
  56         self.no_highlight = no_highlight
  57         self.stream = io.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
  58         if existing_filename:
  59             self.source_stream = io.open(existing_filename, encoding='utf-8')
  60         else:
  61             self.source_stream = None
  62             print(u'''
  63 <html>
  64 <h1>Unable to locate file {}</h1>
  65 </html>
  66             '''.format(filename), file=self.stream)
  67
  68         self.html_formatter = HtmlFormatter(encoding='utf-8')
  69         self.cpp_lexer = CppLexer(stripnl=False)
  70
  71     def render_source_lines(self, stream, line_remarks):
  72         file_text = stream.read()
  73
  74         if self.no_highlight:
  75             html_highlighted = file_text
  76         else:
  77             html_highlighted = highlight(
  78             file_text,
  79                 self.cpp_lexer,
  80                 self.html_formatter)
  81
  82             # Note that the API is different between Python 2 and 3.  On
  83             # Python 3, pygments.highlight() returns a bytes object, so we
  84             # have to decode.  On Python 2, the output is str but since we
  85             # support unicode characters and the output streams is unicode we
  86             # decode too.
  87             html_highlighted = html_highlighted.decode('utf-8')
  88
  89             # Take off the header and footer, these must be
  90             #   reapplied line-wise, within the page structure
  91             html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
  92             html_highlighted = html_highlighted.replace('</pre></div>', '')
  93
  94         for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
  95             print(u'''
  96 <tr>
  97 <td><a name=\"L{linenum}\">{linenum}</a></td>
  98 <td></td>
  99 <td></td>
 100 <td><div class="highlight"><pre>{html_line}</pre></div></td>
 101 </tr>'''.format(**locals()), file=self.stream)
 102
 103             for remark in line_remarks.get(linenum, []):
 104                 if not suppress(remark):
 105                     self.render_inline_remarks(remark, html_line)
 106
 107     def render_inline_remarks(self, r, line):
 108         inlining_context = r.DemangledFunctionName
 109         dl = context.caller_loc.get(r.Function)
 110         if dl:
 111             dl_dict = dict(list(dl))
 112             link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
 113             inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
 114
 115         # Column is the number of characters *including* tabs, keep those and
 116         # replace everything else with spaces.
 117         indent = line[:max(r.Column, 1) - 1]
 118         indent = re.sub('\S', ' ', indent)
 119
 120         # Create expanded message and link if we have a multiline message.
 121         lines = r.message.split('\n')
 122         if len(lines) > 1:
 123             expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
 124             message = lines[0]
 125             expand_message = u'''
 126 <div class="full-info" style="display:none;">
 127   <div class="col-left"><pre style="display:inline">{}</pre></div>
 128   <div class="expanded col-left"><pre>{}</pre></div>
 129 </div>'''.format(indent, '\n'.join(lines[1:]))
 130         else:
 131             expand_link = ''
 132             expand_message = ''
 133             message = r.message
 134         print(u'''
 135 <tr>
 136 <td></td>
 137 <td>{r.RelativeHotness}</td>
 138 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 139 <td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message}&nbsp;</span>{expand_message}</td>
 140 <td class=\"column-entry-yellow\">{inlining_context}</td>
 141 </tr>'''.format(**locals()), file=self.stream)
 142
 143     def render(self, line_remarks):
 144         if not self.source_stream:
 145             return
 146
 147         print(u'''
 148 <html>
 149 <title>{}</title>
 150 <meta charset="utf-8" />
 151 <head>
 152 <link rel='stylesheet' type='text/css' href='style.css'>
 153 <script type="text/javascript">
 154 /* Simple helper to show/hide the expanded message of a remark. */
 155 function toggleExpandedMessage(e) {{
 156   var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
 157   if (!FullTextElems || FullTextElems.length < 1) {{
 158       return false;
 159   }}
 160   var FullText = FullTextElems[0];
 161   if (FullText.style.display == 'none') {{
 162     e.innerHTML = '-';
 163     FullText.style.display = 'block';
 164   }} else {{
 165     e.innerHTML = '+';
 166     FullText.style.display = 'none';
 167   }}
 168 }}
 169 </script>
 170 </head>
 171 <body>
 172 <div class="centered">
 173 <table class="source">
 174 <thead>
 175 <tr>
 176 <th style="width: 2%">Line</td>
 177 <th style="width: 3%">Hotness</td>
 178 <th style="width: 10%">Optimization</td>
 179 <th style="width: 70%">Source</td>
 180 <th style="width: 15%">Inline Context</td>
 181 </tr>
 182 </thead>
 183 <tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
 184         self.render_source_lines(self.source_stream, line_remarks)
 185
 186         print(u'''
 187 </tbody>
 188 </table>
 189 </body>
 190 </html>''', file=self.stream)
 191
 192
 193 class IndexRenderer:
 194     def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
 195         self.stream = io.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
 196         self.should_display_hotness = should_display_hotness
 197         self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
 198
 199     def render_entry(self, r, odd):
 200         escaped_name = html.escape(r.DemangledFunctionName)
 201         print(u'''
 202 <tr>
 203 <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
 204 <td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
 205 <td class=\"column-entry-{odd}\">{escaped_name}</td>
 206 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 207 </tr>'''.format(**locals()), file=self.stream)
 208
 209     def render(self, all_remarks):
 210         print(u'''
 211 <html>
 212 <meta charset="utf-8" />
 213 <head>
 214 <link rel='stylesheet' type='text/css' href='style.css'>
 215 </head>
 216 <body>
 217 <div class="centered">
 218 <table>
 219 <tr>
 220 <td>Source Location</td>
 221 <td>Hotness</td>
 222 <td>Function</td>
 223 <td>Pass</td>
 224 </tr>''', file=self.stream)
 225
 226         max_entries = None
 227         if self.should_display_hotness:
 228             max_entries = self.max_hottest_remarks_on_index
 229
 230         for i, remark in enumerate(all_remarks[:max_entries]):
 231             if not suppress(remark):
 232                 self.render_entry(remark, i % 2)
 233         print(u'''
 234 </table>
 235 </body>
 236 </html>''', file=self.stream)
 237
 238
 239 def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
 240     global context
 241     context = ctx
 242     filename, remarks = entry
 243     SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
 244
 245
 246 def map_remarks(all_remarks):
 247     # Set up a map between function names and their source location for
 248     # function where inlining happened
 249     for remark in optrecord.itervalues(all_remarks):
 250         if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
 251             for arg in remark.Args:
 252                 arg_dict = dict(list(arg))
 253                 caller = arg_dict.get('Caller')
 254                 if caller:
 255                     try:
 256                         context.caller_loc[caller] = arg_dict['DebugLoc']
 257                     except KeyError:
 258                         pass
 259
 260
 261 def generate_report(all_remarks,
 262                     file_remarks,
 263                     source_dir,
 264                     output_dir,
 265                     no_highlight,
 266                     should_display_hotness,
 267                     max_hottest_remarks_on_index,
 268                     num_jobs,
 269                     should_print_progress):
 270     try:
 271         os.makedirs(output_dir)
 272     except OSError as e:
 273         if e.errno == errno.EEXIST and os.path.isdir(output_dir):
 274             pass
 275         else:
 276             raise
 277
 278     if should_print_progress:
 279         print('Rendering index page...')
 280     if should_display_hotness:
 281         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
 282     else:
 283         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
 284     IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
 285
 286     shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
 287             "style.css"), output_dir)
 288
 289     _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
 290     if should_print_progress:
 291         print('Rendering HTML files...')
 292     optpmap.pmap(_render_file_bound,
 293                  file_remarks.items(),
 294                  num_jobs,
 295                  should_print_progress)
 296
 297
 298 def main():
 299     parser = argparse.ArgumentParser(description=desc)
 300     parser.add_argument(
 301         'yaml_dirs_or_files',
 302         nargs='+',
 303         help='List of optimization record files or directories searched '
 304              'for optimization record files.')
 305     parser.add_argument(
 306         '--output-dir',
 307         '-o',
 308         default='html',
 309         help='Path to a directory where generated HTML files will be output. '
 310              'If the directory does not already exist, it will be created. '
 311              '"%(default)s" by default.')
 312     parser.add_argument(
 313         '--jobs',
 314         '-j',
 315         default=None,
 316         type=int,
 317         help='Max job count (defaults to %(default)s, the current CPU count)')
 318     parser.add_argument(
 319         '--source-dir',
 320         '-s',
 321         default='',
 322         help='set source directory')
 323     parser.add_argument(
 324         '--no-progress-indicator',
 325         '-n',
 326         action='store_true',
 327         default=False,
 328         help='Do not display any indicator of how many YAML files were read '
 329              'or rendered into HTML.')
 330     parser.add_argument(
 331         '--max-hottest-remarks-on-index',
 332         default=1000,
 333         type=int,
 334         help='Maximum number of the hottest remarks to appear on the index page')
 335     parser.add_argument(
 336         '--no-highlight',
 337         action='store_true',
 338         default=False,
 339         help='Do not use a syntax highlighter when rendering the source code')
 340     parser.add_argument(
 341         '--demangler',
 342         help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
 343
 344     parser.add_argument(
 345         '--filter',
 346         default='',
 347         help='Only display remarks from passes matching filter expression')
 348
 349     # Do not make this a global variable.  Values needed to be propagated through
 350     # to individual classes and functions to be portable with multiprocessing across
 351     # Windows and non-Windows.
 352     args = parser.parse_args()
 353
 354     print_progress = not args.no_progress_indicator
 355     if args.demangler:
 356         optrecord.Remark.set_demangler(args.demangler)
 357
 358     files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
 359     if not files:
 360         parser.error("No *.opt.yaml files found")
 361         sys.exit(1)
 362
 363     all_remarks, file_remarks, should_display_hotness = \
 364         optrecord.gather_results(files, args.jobs, print_progress, args.filter)
 365
 366     map_remarks(all_remarks)
 367
 368     generate_report(all_remarks,
 369                     file_remarks,
 370                     args.source_dir,
 371                     args.output_dir,
 372                     args.no_highlight,
 373                     should_display_hotness,
 374                     args.max_hottest_remarks_on_index,
 375                     args.jobs,
 376                     print_progress)
 377
 378 if __name__ == '__main__':
 379     main()