docs/doxygen/includesorter.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2012,2013,2014,2015, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """Include directive sorter for GROMACS.
  37
  38 This module implements an #include directive sorter for GROMACS C/C++ files.
  39 It allows (in most cases) automatically sorting includes and formatting
  40 the paths to use either relative paths or paths relative to src/.
  41 It groups includes in groups of related headers, sorts the headers
  42 alphabetically within each block, and inserts empty lines in between.
  43 It can be run as a standalone script, in which case it requires an up-to-date
  44 list of installed headers and Doxygen XML documentation to be present in the
  45 build tree.  It can also be imported as a module to be embedded in other
  46 scripts.  In the latter case, the IncludeSorter provides the main interface.
  47
  48 The sorting assumes some conventions (e.g., that system headers are included
  49 with angle brackets instead of quotes).  Generally, these conventions are
  50 checked by the check-source.py script.
  51 """
  52
  53 import os.path
  54 import re
  55 import sys
  56
  57 class IncludeGroup(object):
  58
  59     """Enumeration type for grouping includes."""
  60
  61     def __init__(self, value):
  62         """Initialize a IncludeGroup instance.
  63
  64         IncludeGroup.{main,system_c,...} should be used outside the
  65         class instead of calling the constructor.
  66         """
  67         self._value = value
  68
  69     def __cmp__(self, other):
  70         """Order include groups in the desired order."""
  71         return cmp(self._value, other._value)
  72
  73 # gmxpre.h is always first
  74 IncludeGroup.pre = IncludeGroup(0)
  75 # "main" include file for the source file is next
  76 IncludeGroup.main = IncludeGroup(1)
  77 # config.h is next, if present, to keep its location consistent
  78 IncludeGroup.config = IncludeGroup(2)
  79 # Followed by system headers, with C first and C++ following
  80 IncludeGroup.system_c = IncludeGroup(3)
  81 IncludeGroup.system_c_cpp = IncludeGroup(4)
  82 IncludeGroup.system_cpp = IncludeGroup(5)
  83 # System headers not in standard C/C++ are in a separate block
  84 IncludeGroup.system_other = IncludeGroup(6)
  85 # src/external/ contents that are included with quotes go here
  86 IncludeGroup.nonsystem_other = IncludeGroup(7)
  87 # Other GROMACS headers
  88 IncludeGroup.gmx_general = IncludeGroup(8)
  89 # This group is for shared (unit) testing utilities
  90 IncludeGroup.gmx_test = IncludeGroup(9)
  91 # This group is for headers local to the including file/module
  92 IncludeGroup.gmx_local = IncludeGroup(10)
  93
  94 class GroupedSorter(object):
  95
  96     """Grouping and formatting logic for #include directives.
  97
  98     This class implements the actual logic that decides how includes are
  99     grouped and sorted, and how they are formatted."""
 100
 101     # These variables contain the list of system headers for various blocks
 102     _std_c_headers = ['assert.h', 'ctype.h', 'errno.h', 'float.h',
 103             'inttypes.h', 'limits.h', 'math.h', 'signal.h', 'stdarg.h',
 104             'stddef.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h',
 105             'time.h']
 106     _std_c_cpp_headers = ['c' + x[:-2] for x in _std_c_headers]
 107     _std_cpp_headers = ['algorithm', 'deque', 'exception', 'fstream',
 108             'iomanip', 'ios', 'iosfwd', 'iostream', 'istream', 'iterator',
 109             'limits', 'list', 'map', 'memory', 'new', 'numeric', 'ostream',
 110             'regex', 'set', 'sstream', 'stdexcept', 'streambuf', 'string', 'strstream',
 111             'typeinfo', 'vector', 'utility']
 112
 113     def __init__(self, style='pub-priv', absolute=False):
 114         """Initialize a sorted with the given style."""
 115         if style == 'single-group':
 116             self._local_group = 'none'
 117         elif style == 'pub-priv':
 118             self._local_group = 'private'
 119         else:
 120             self._local_group = 'local'
 121         if absolute:
 122             self._abspath_main = True
 123             self._abspath_local = True
 124         else:
 125             self._abspath_main = False
 126             self._abspath_local = False
 127
 128     def _get_path(self, included_file, group, including_file):
 129         """Compute include path to use for an #include.
 130
 131         The path is made either absolute (i.e., relative to src/), or
 132         relative to the location of the including file, depending on the group
 133         the file is in.
 134         """
 135         use_abspath = including_file is None or group is None
 136         if not use_abspath:
 137             if group in (IncludeGroup.gmx_general, IncludeGroup.gmx_test):
 138                 use_abspath = True
 139             elif group == IncludeGroup.main and self._abspath_main:
 140                 use_abspath = True
 141             elif group == IncludeGroup.gmx_local and self._abspath_local:
 142                 use_abspath = True
 143         if not use_abspath:
 144             fromdir = os.path.dirname(including_file.get_abspath())
 145             relpath = os.path.relpath(included_file.get_abspath(), fromdir)
 146             if not relpath.startswith('..'):
 147                 return relpath
 148         path = included_file.get_relpath()
 149         assert path.startswith('src/')
 150         return path[4:]
 151
 152     def _get_gmx_group(self, including_file, included_file):
 153         """Determine group for GROMACS headers.
 154
 155         Helper function to determine the group for an #include directive
 156         when the #include is in one of the gmx_* groups (or in the main group).
 157         """
 158         main_header = including_file.get_main_header()
 159         if main_header and main_header == included_file:
 160             return IncludeGroup.main
 161         if included_file.get_directory().get_name() == 'testutils':
 162             return IncludeGroup.gmx_test
 163         if including_file.get_directory().contains(included_file):
 164             if self._local_group == 'local':
 165                 return IncludeGroup.gmx_local
 166             if self._local_group == 'private':
 167                 if included_file.api_type_is_reliable() \
 168                         and included_file.is_module_internal():
 169                     return IncludeGroup.gmx_local
 170                 if not included_file.api_type_is_reliable() \
 171                         and including_file.get_relpath().startswith('src/programs'):
 172                     return IncludeGroup.gmx_local
 173         if included_file.is_test_file():
 174             return IncludeGroup.gmx_test
 175         return IncludeGroup.gmx_general
 176
 177     def get_sortable_object(self, include):
 178         """Produce a sortable, opaque object for an include.
 179
 180         Includes are sorted by calling this function for each #include object,
 181         and sorting the list made up of these objects (using the default
 182         comparison operators).  Each element from the sorted list is then
 183         passed to format_include(), which extracts information from the opaque
 184         object and formats the #include directive for output.
 185         """
 186         included_file = include.get_file()
 187         if not included_file:
 188             path = include.get_included_path()
 189             if path in self._std_c_headers:
 190                 group = IncludeGroup.system_c
 191             elif path in self._std_c_cpp_headers:
 192                 group = IncludeGroup.system_c_cpp
 193             elif path in self._std_cpp_headers:
 194                 group = IncludeGroup.system_cpp
 195             else:
 196                 group = IncludeGroup.system_other
 197         elif included_file.is_external():
 198             group = IncludeGroup.nonsystem_other
 199             if 'external/' in include.get_included_path():
 200                 path = self._get_path(included_file, group, None)
 201             else:
 202                 path = include.get_included_path()
 203         elif included_file.get_name() == 'gmxpre.h':
 204             group = IncludeGroup.pre
 205             path = self._get_path(included_file, group, None)
 206         elif included_file.get_name() == 'config.h':
 207             group = IncludeGroup.config
 208             path = self._get_path(included_file, group, None)
 209         else:
 210             including_file = include.get_including_file()
 211             group = self._get_gmx_group(including_file, included_file)
 212             path = self._get_path(included_file, group, including_file)
 213         return (group, os.path.split(path), include)
 214
 215     def format_include(self, obj, prev):
 216         """Format an #include directive after sorting."""
 217         result = []
 218         if prev:
 219             if prev[0] != obj[0]:
 220                 # Print empty line between groups
 221                 result.append('\n')
 222             elif prev[1] == obj[1]:
 223                 # Skip duplicates
 224                 return result
 225         include = obj[2]
 226         line = include.get_full_line()
 227         include_re = r'^(?P<head>\s*#\s*include\s+)["<][^">]*[">](?P<tail>.*)$'
 228         match = re.match(include_re, line)
 229         assert match
 230         if include.is_system():
 231             path = '<{0}>'.format(os.path.join(obj[1][0], obj[1][1]))
 232         else:
 233             path = '"{0}"'.format(os.path.join(obj[1][0], obj[1][1]))
 234         result.append('{0}{1}{2}\n'.format(match.group('head'), path, match.group('tail')))
 235         return result
 236
 237 class IncludeSorter(object):
 238
 239     """High-level logic for sorting includes.
 240
 241     This class contains the high-level logic for sorting include statements.
 242     The actual ordering and formatting the includes is delegated to a sort method
 243     (see GroupedSorter) to keep things separated.
 244     """
 245
 246     def __init__(self, sortmethod=None, quiet=True):
 247         """Initialize the include sorter with the given sorter and options."""
 248         if not sortmethod:
 249             sortmethod = GroupedSorter()
 250         self._sortmethod = sortmethod
 251         self._quiet = quiet
 252         self._changed = False
 253
 254     def _sort_include_block(self, block, lines):
 255         """Sort a single include block.
 256
 257         Returns a new list of lines for the block.
 258         If anything is changed, self._changed is set to True, and the caller
 259         can check that."""
 260         includes = map(self._sortmethod.get_sortable_object, block.get_includes())
 261         includes.sort()
 262         result = []
 263         prev = None
 264         current_line_number = block.get_first_line()-1
 265         for include in includes:
 266             newlines = self._sortmethod.format_include(include, prev)
 267             result.extend(newlines)
 268             if not self._changed:
 269                 for offset, newline in enumerate(newlines):
 270                     if lines[current_line_number + offset] != newline:
 271                         self._changed = True
 272                         break
 273                 current_line_number += len(newlines)
 274             prev = include
 275         return result
 276
 277     def sort_includes(self, fileobj):
 278         """Sort all includes in a file."""
 279         lines = fileobj.get_contents()
 280         # Format into a list first:
 281         #  - avoid bugs or issues in the script truncating the file
 282         #  - can check whether anything was changed before touching the file
 283         newlines = []
 284         prev = 0
 285         self._changed = False
 286         for block in fileobj.get_include_blocks():
 287             newlines.extend(lines[prev:block.get_first_line()-1])
 288             newlines.extend(self._sort_include_block(block, lines))
 289             # The returned values are 1-based, but indexing here is 0-based,
 290             # so an explicit +1 is not needed.
 291             prev = block.get_last_line()
 292         if self._changed:
 293             if not self._quiet:
 294                 sys.stderr.write('{0}: includes reformatted\n'.format(fileobj.get_relpath()))
 295             newlines.extend(lines[prev:])
 296             with open(fileobj.get_abspath(), 'w') as fp:
 297                 fp.write(''.join(newlines))
 298
 299     def check_sorted(self, fileobj):
 300         """Check that includes within a file are sorted."""
 301         # TODO: Make the checking work without full contents of the file
 302         lines = fileobj.get_contents()
 303         is_sorted = True
 304         details = None
 305         for block in fileobj.get_include_blocks():
 306             self._changed = False
 307             sorted_lines = self._sort_include_block(block, lines)
 308             if self._changed:
 309                 is_sorted = False
 310                 # TODO: Do a proper diff to show the actual changes.
 311                 if details is None:
 312                     details = ["Correct order/style is:"]
 313                 else:
 314                     details.append("    ...")
 315                 details.extend(["    " + x.rstrip() for x in sorted_lines])
 316         return (is_sorted, details)
 317
 318 def main():
 319     """Run the include sorter script."""
 320     import os
 321     import sys
 322
 323     from optparse import OptionParser
 324
 325     from gmxtree import GromacsTree
 326     from reporter import Reporter
 327
 328     parser = OptionParser()
 329     parser.add_option('-S', '--source-root',
 330                       help='Source tree root directory')
 331     parser.add_option('-B', '--build-root',
 332                       help='Build tree root directory')
 333     parser.add_option('-F', '--files',
 334                       help='Specify files to sort')
 335     parser.add_option('-q', '--quiet', action='store_true',
 336                       help='Do not write status messages')
 337     # This is for evaluating different options; can be removed from the final
 338     # version.
 339     parser.add_option('-s', '--style', type='choice', default='pub-priv',
 340                       choices=('single-group', 'pub-priv', 'pub-local'),
 341                       help='Style for GROMACS includes')
 342     parser.add_option('--absolute', action='store_true',
 343                       help='Write all include paths relative to src/')
 344     options, args = parser.parse_args()
 345
 346     filelist = args
 347     if options.files:
 348         if options.files == '-':
 349             lines = sys.stdin.readlines()
 350         else:
 351             with open(options.files, 'r') as fp:
 352                 lines = fp.readlines()
 353         filelist.extend([x.strip() for x in lines])
 354
 355     reporter = Reporter(quiet=True)
 356
 357     if not options.quiet:
 358         sys.stderr.write('Scanning source tree...\n')
 359     tree = GromacsTree(options.source_root, options.build_root, reporter)
 360     tree.load_installed_file_list()
 361     files = []
 362     for filename in filelist:
 363         fileobj = tree.get_file(os.path.abspath(filename))
 364         if not fileobj:
 365             sys.stderr.write('warning: ignoring unknown file {0}\n'.format(filename))
 366             continue
 367         files.append(fileobj)
 368     if not options.quiet:
 369         sys.stderr.write('Reading source files...\n')
 370     tree.scan_files(only_files=files, keep_contents=True)
 371     extfiles = set(files)
 372     for fileobj in files:
 373         for included_file in fileobj.get_includes():
 374             other_file = included_file.get_file()
 375             if other_file:
 376                 extfiles.add(other_file)
 377     if not options.quiet:
 378         sys.stderr.write('Reading Doxygen XML files...\n')
 379     tree.load_xml(only_files=extfiles)
 380
 381     if not options.quiet:
 382         sys.stderr.write('Sorting includes...\n')
 383
 384     sorter = IncludeSorter(GroupedSorter(options.style, options.absolute), options.quiet)
 385
 386     for fileobj in files:
 387         sorter.sort_includes(fileobj)
 388
 389 if __name__ == '__main__':
 390     main()