tools/opt-viewer/opt-viewer.py

   1 #!/usr/bin/env python2.7
   2
   3 from __future__ import print_function
   4
   5 import argparse
   6 import cgi
   7 import codecs
   8 import errno
   9 import functools
  10 from multiprocessing import cpu_count
  11 import os.path
  12 import re
  13 import shutil
  14 import sys
  15
  16 from pygments import highlight
  17 from pygments.lexers.c_cpp import CppLexer
  18 from pygments.formatters import HtmlFormatter
  19
  20 import optpmap
  21 import optrecord
  22
  23
  24 desc = '''Generate HTML output to visualize optimization records from the YAML files
  25 generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
  26
  27 The tools requires PyYAML and Pygments Python packages.'''
  28
  29
  30 # This allows passing the global context to the child processes.
  31 class Context:
  32     def __init__(self, caller_loc = dict()):
  33        # Map function names to their source location for function where inlining happened
  34        self.caller_loc = caller_loc
  35
  36 context = Context()
  37
  38 def suppress(remark):
  39     if remark.Name == 'sil.Specialized':
  40         return remark.getArgDict()['Function'][0].startswith('\"Swift.')
  41     elif remark.Name == 'sil.Inlined':
  42         return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
  43     return False
  44
  45 class SourceFileRenderer:
  46     def __init__(self, source_dir, output_dir, filename, no_highlight):
  47         self.filename = filename
  48         existing_filename = None
  49         if os.path.exists(filename):
  50             existing_filename = filename
  51         else:
  52             fn = os.path.join(source_dir, filename)
  53             if os.path.exists(fn):
  54                 existing_filename = fn
  55
  56         self.no_highlight = no_highlight
  57         self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
  58         if existing_filename:
  59             self.source_stream = open(existing_filename)
  60         else:
  61             self.source_stream = None
  62             print('''
  63 <html>
  64 <h1>Unable to locate file {}</h1>
  65 </html>
  66             '''.format(filename), file=self.stream)
  67
  68         self.html_formatter = HtmlFormatter(encoding='utf-8')
  69         self.cpp_lexer = CppLexer(stripnl=False)
  70
  71     def render_source_lines(self, stream, line_remarks):
  72         file_text = stream.read()
  73
  74         if self.no_highlight:
  75             html_highlighted = file_text.decode('utf-8')
  76         else:
  77             html_highlighted = highlight(
  78             file_text,
  79                 self.cpp_lexer,
  80                 self.html_formatter)
  81
  82             # Note that the API is different between Python 2 and 3.  On
  83             # Python 3, pygments.highlight() returns a bytes object, so we
  84             # have to decode.  On Python 2, the output is str but since we
  85             # support unicode characters and the output streams is unicode we
  86             # decode too.
  87             html_highlighted = html_highlighted.decode('utf-8')
  88
  89             # Take off the header and footer, these must be
  90             #   reapplied line-wise, within the page structure
  91             html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
  92             html_highlighted = html_highlighted.replace('</pre></div>', '')
  93
  94         for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
  95             print(u'''
  96 <tr>
  97 <td><a name=\"L{linenum}\">{linenum}</a></td>
  98 <td></td>
  99 <td></td>
 100 <td><div class="highlight"><pre>{html_line}</pre></div></td>
 101 </tr>'''.format(**locals()), file=self.stream)
 102
 103             for remark in line_remarks.get(linenum, []):
 104                 if not suppress(remark):
 105                     self.render_inline_remarks(remark, html_line)
 106
 107     def render_inline_remarks(self, r, line):
 108         inlining_context = r.DemangledFunctionName
 109         dl = context.caller_loc.get(r.Function)
 110         if dl:
 111             dl_dict = dict(list(dl))
 112             link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
 113             inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
 114
 115         # Column is the number of characters *including* tabs, keep those and
 116         # replace everything else with spaces.
 117         indent = line[:max(r.Column, 1) - 1]
 118         indent = re.sub('\S', ' ', indent)
 119
 120         print(u'''
 121 <tr>
 122 <td></td>
 123 <td>{r.RelativeHotness}</td>
 124 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 125 <td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\"> {r.message}&nbsp;</span></td>
 126 <td class=\"column-entry-yellow\">{inlining_context}</td>
 127 </tr>'''.format(**locals()), file=self.stream)
 128
 129     def render(self, line_remarks):
 130         if not self.source_stream:
 131             return
 132
 133         print('''
 134 <html>
 135 <title>{}</title>
 136 <meta charset="utf-8" />
 137 <head>
 138 <link rel='stylesheet' type='text/css' href='style.css'>
 139 </head>
 140 <body>
 141 <div class="centered">
 142 <table class="source">
 143 <thead>
 144 <tr>
 145 <th style="width: 2%">Line</td>
 146 <th style="width: 3%">Hotness</td>
 147 <th style="width: 10%">Optimization</td>
 148 <th style="width: 70%">Source</td>
 149 <th style="width: 15%">Inline Context</td>
 150 </tr>
 151 </thead>
 152 <tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
 153         self.render_source_lines(self.source_stream, line_remarks)
 154
 155         print('''
 156 </tbody>
 157 </table>
 158 </body>
 159 </html>''', file=self.stream)
 160
 161
 162 class IndexRenderer:
 163     def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
 164         self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
 165         self.should_display_hotness = should_display_hotness
 166         self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
 167
 168     def render_entry(self, r, odd):
 169         escaped_name = cgi.escape(r.DemangledFunctionName)
 170         print(u'''
 171 <tr>
 172 <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
 173 <td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
 174 <td class=\"column-entry-{odd}\">{escaped_name}</td>
 175 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
 176 </tr>'''.format(**locals()), file=self.stream)
 177
 178     def render(self, all_remarks):
 179         print('''
 180 <html>
 181 <meta charset="utf-8" />
 182 <head>
 183 <link rel='stylesheet' type='text/css' href='style.css'>
 184 </head>
 185 <body>
 186 <div class="centered">
 187 <table>
 188 <tr>
 189 <td>Source Location</td>
 190 <td>Hotness</td>
 191 <td>Function</td>
 192 <td>Pass</td>
 193 </tr>''', file=self.stream)
 194
 195         max_entries = None
 196         if self.should_display_hotness:
 197             max_entries = self.max_hottest_remarks_on_index
 198
 199         for i, remark in enumerate(all_remarks[:max_entries]):
 200             if not suppress(remark):
 201                 self.render_entry(remark, i % 2)
 202         print('''
 203 </table>
 204 </body>
 205 </html>''', file=self.stream)
 206
 207
 208 def _render_file(source_dir, output_dir, ctx, no_highlight, entry):
 209     global context
 210     context = ctx
 211     filename, remarks = entry
 212     SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
 213
 214
 215 def map_remarks(all_remarks):
 216     # Set up a map between function names and their source location for
 217     # function where inlining happened
 218     for remark in optrecord.itervalues(all_remarks):
 219         if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
 220             for arg in remark.Args:
 221                 arg_dict = dict(list(arg))
 222                 caller = arg_dict.get('Caller')
 223                 if caller:
 224                     try:
 225                         context.caller_loc[caller] = arg_dict['DebugLoc']
 226                     except KeyError:
 227                         pass
 228
 229
 230 def generate_report(all_remarks,
 231                     file_remarks,
 232                     source_dir,
 233                     output_dir,
 234                     no_highlight,
 235                     should_display_hotness,
 236                     max_hottest_remarks_on_index,
 237                     num_jobs,
 238                     should_print_progress):
 239     try:
 240         os.makedirs(output_dir)
 241     except OSError as e:
 242         if e.errno == errno.EEXIST and os.path.isdir(output_dir):
 243             pass
 244         else:
 245             raise
 246
 247     if should_print_progress:
 248         print('Rendering index page...')
 249     if should_display_hotness:
 250         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
 251     else:
 252         sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
 253     IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
 254
 255     shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
 256             "style.css"), output_dir)
 257
 258     _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
 259     if should_print_progress:
 260         print('Rendering HTML files...')
 261     optpmap.pmap(_render_file_bound,
 262                  file_remarks.items(),
 263                  num_jobs,
 264                  should_print_progress)
 265
 266
 267 def main():
 268     parser = argparse.ArgumentParser(description=desc)
 269     parser.add_argument(
 270         'yaml_dirs_or_files',
 271         nargs='+',
 272         help='List of optimization record files or directories searched '
 273              'for optimization record files.')
 274     parser.add_argument(
 275         '--output-dir',
 276         '-o',
 277         default='html',
 278         help='Path to a directory where generated HTML files will be output. '
 279              'If the directory does not already exist, it will be created. '
 280              '"%(default)s" by default.')
 281     parser.add_argument(
 282         '--jobs',
 283         '-j',
 284         default=None,
 285         type=int,
 286         help='Max job count (defaults to %(default)s, the current CPU count)')
 287     parser.add_argument(
 288         '--source-dir',
 289         '-s',
 290         default='',
 291         help='set source directory')
 292     parser.add_argument(
 293         '--no-progress-indicator',
 294         '-n',
 295         action='store_true',
 296         default=False,
 297         help='Do not display any indicator of how many YAML files were read '
 298              'or rendered into HTML.')
 299     parser.add_argument(
 300         '--max-hottest-remarks-on-index',
 301         default=1000,
 302         type=int,
 303         help='Maximum number of the hottest remarks to appear on the index page')
 304     parser.add_argument(
 305         '--no-highlight',
 306         action='store_true',
 307         default=False,
 308         help='Do not use a syntax highlighter when rendering the source code')
 309     parser.add_argument(
 310         '--demangler',
 311         help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
 312
 313     # Do not make this a global variable.  Values needed to be propagated through
 314     # to individual classes and functions to be portable with multiprocessing across
 315     # Windows and non-Windows.
 316     args = parser.parse_args()
 317
 318     print_progress = not args.no_progress_indicator
 319     if args.demangler:
 320         optrecord.Remark.set_demangler(args.demangler)
 321
 322     files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
 323     if not files:
 324         parser.error("No *.opt.yaml files found")
 325         sys.exit(1)
 326
 327     all_remarks, file_remarks, should_display_hotness = \
 328         optrecord.gather_results(files, args.jobs, print_progress)
 329
 330     map_remarks(all_remarks)
 331
 332     generate_report(all_remarks,
 333                     file_remarks,
 334                     args.source_dir,
 335                     args.output_dir,
 336                     args.no_highlight,
 337                     should_display_hotness,
 338                     args.max_hottest_remarks_on_index,
 339                     args.jobs,
 340                     print_progress)
 341
 342 if __name__ == '__main__':
 343     main()