docs/doxygen/includesorter.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """Include directive sorter for GROMACS.
  37
  38 This module implements an #include directive sorter for GROMACS C/C++ files.
  39 It allows (in most cases) automatically sorting includes and formatting
  40 the paths to use either relative paths or paths relative to src/.
  41 It groups includes in groups of related headers, sorts the headers
  42 alphabetically within each block, and inserts empty lines in between.
  43 It can be run as a standalone script, in which case it requires an up-to-date
  44 list of installed headers and Doxygen XML documentation to be present in the
  45 build tree.  It can also be imported as a module to be embedded in other
  46 scripts.  In the latter case, the IncludeSorter provides the main interface.
  47
  48 The sorting assumes some conventions (e.g., that system headers are included
  49 with angle brackets instead of quotes).  Generally, these conventions are
  50 checked by the check-source.py script.
  51 """
  52
  53 import os.path
  54 import re
  55 import sys
  56
  57 class IncludeGroup(object):
  58
  59     """Enumeration type for grouping includes."""
  60
  61     def __init__(self, value):
  62         """Initialize a IncludeGroup instance.
  63
  64         IncludeGroup.{main,system_c,...} should be used outside the
  65         class instead of calling the constructor.
  66         """
  67         self._value = value
  68
  69     def __cmp__(self, other):
  70         """Order include groups in the desired order."""
  71         return cmp(self._value, other._value)
  72
  73 # gmxpre.h is always first
  74 IncludeGroup.pre = IncludeGroup(0)
  75 # "main" include file for the source file is next
  76 IncludeGroup.main = IncludeGroup(1)
  77 # config.h is next, if present, to keep its location consistent
  78 IncludeGroup.config = IncludeGroup(2)
  79 # Followed by system headers, with C first and C++ following
  80 IncludeGroup.system_c = IncludeGroup(3)
  81 IncludeGroup.system_c_cpp = IncludeGroup(4)
  82 IncludeGroup.system_cpp = IncludeGroup(5)
  83 # System headers not in standard C/C++ are in a separate block
  84 IncludeGroup.system_other = IncludeGroup(6)
  85 # src/external/ contents that are included with quotes go here
  86 IncludeGroup.nonsystem_other = IncludeGroup(7)
  87 # Other GROMACS headers
  88 IncludeGroup.gmx_general = IncludeGroup(8)
  89 # This group is for shared (unit) testing utilities
  90 IncludeGroup.gmx_test = IncludeGroup(9)
  91 # This group is for headers local to the including file/module
  92 IncludeGroup.gmx_local = IncludeGroup(10)
  93
  94 class GroupedSorter(object):
  95
  96     """Grouping and formatting logic for #include directives.
  97
  98     This class implements the actual logic that decides how includes are
  99     grouped and sorted, and how they are formatted."""
 100
 101     # These variables contain the list of system headers for various blocks
 102     _std_c_headers = ['assert.h', 'ctype.h', 'errno.h', 'float.h',
 103             'inttypes.h', 'limits.h', 'math.h', 'signal.h', 'stdarg.h',
 104             'stddef.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h',
 105             'time.h']
 106     _std_c_cpp_headers = ['c' + x[:-2] for x in _std_c_headers]
 107     _std_cpp_headers = ['algorithm', 'array', 'deque', 'exception', 'fstream',
 108             'functional', 'iomanip', 'ios', 'iosfwd', 'iostream', 'istream', 'iterator',
 109             'limits', 'list', 'map', 'memory', 'new', 'numeric', 'ostream', 'random',
 110             'regex', 'set', 'sstream', 'stdexcept', 'streambuf', 'string', 'strstream',
 111             'tuple', 'type_traits', 'typeindex', 'typeinfo', 'vector', 'utility']
 112
 113     def __init__(self, style='pub-priv', absolute=False):
 114         """Initialize a sorted with the given style."""
 115         if style == 'single-group':
 116             self._local_group = 'none'
 117         elif style == 'pub-priv':
 118             self._local_group = 'private'
 119         else:
 120             self._local_group = 'local'
 121         if absolute:
 122             self._abspath_main = True
 123             self._abspath_local = True
 124         else:
 125             self._abspath_main = False
 126             self._abspath_local = False
 127
 128     def _get_path(self, included_file, group, including_file):
 129         """Compute include path to use for an #include.
 130
 131         The path is made either absolute (i.e., relative to src/), or
 132         relative to the location of the including file, depending on the group
 133         the file is in.
 134         """
 135         use_abspath = including_file is None or group is None
 136         if not use_abspath:
 137             if group in (IncludeGroup.gmx_general, IncludeGroup.gmx_test):
 138                 use_abspath = True
 139             elif group == IncludeGroup.main and self._abspath_main:
 140                 use_abspath = True
 141             elif group == IncludeGroup.gmx_local and self._abspath_local:
 142                 use_abspath = True
 143         if not use_abspath:
 144             fromdir = os.path.dirname(including_file.get_abspath())
 145             relpath = os.path.relpath(included_file.get_abspath(), fromdir)
 146             if not relpath.startswith('..'):
 147                 return relpath
 148         path = included_file.get_relpath()
 149         assert path.startswith('src/')
 150         return path[4:]
 151
 152     def _get_gmx_group(self, including_file, included_file):
 153         """Determine group for GROMACS headers.
 154
 155         Helper function to determine the group for an #include directive
 156         when the #include is in one of the gmx_* groups (or in the main group).
 157         """
 158         main_header = including_file.get_main_header()
 159         if main_header and main_header == included_file:
 160             return IncludeGroup.main
 161         if included_file.get_directory().get_name() == 'testutils':
 162             return IncludeGroup.gmx_test
 163         if including_file.get_directory().contains(included_file):
 164             if self._local_group == 'local':
 165                 return IncludeGroup.gmx_local
 166             if self._local_group == 'private':
 167                 if included_file.api_type_is_reliable() \
 168                         and included_file.is_module_internal():
 169                     return IncludeGroup.gmx_local
 170                 if not included_file.api_type_is_reliable() \
 171                         and including_file.get_relpath().startswith('src/programs'):
 172                     return IncludeGroup.gmx_local
 173         if included_file.is_test_file():
 174             return IncludeGroup.gmx_test
 175         return IncludeGroup.gmx_general
 176
 177     def _split_path(self, path):
 178         """Split include path into sortable compoments.
 179
 180         Plain string on the full path in the #include directive causes some
 181         unintuitive behavior, so this splits the path into a tuple at
 182         points that allow more natural sorting: primary sort criterion is the
 183         directory name, followed by the basename (without extension) of the
 184         included file.
 185         """
 186         path_components = list(os.path.split(path))
 187         path_components[1] = os.path.splitext(path_components[1])
 188         return tuple(path_components)
 189
 190     def _join_path(self, path_components):
 191         """Reconstruct path from the return value of _split_path."""
 192         return os.path.join(path_components[0], ''.join(path_components[1]))
 193
 194     def get_sortable_object(self, include):
 195         """Produce a sortable, opaque object for an include.
 196
 197         Includes are sorted by calling this function for each #include object,
 198         and sorting the list made up of these objects (using the default
 199         comparison operators).  Each element from the sorted list is then
 200         passed to format_include(), which extracts information from the opaque
 201         object and formats the #include directive for output.
 202         """
 203         included_file = include.get_file()
 204         if not included_file:
 205             path = include.get_included_path()
 206             if path in self._std_c_headers:
 207                 group = IncludeGroup.system_c
 208             elif path in self._std_c_cpp_headers:
 209                 group = IncludeGroup.system_c_cpp
 210             elif path in self._std_cpp_headers:
 211                 group = IncludeGroup.system_cpp
 212             else:
 213                 group = IncludeGroup.system_other
 214         elif included_file.is_external():
 215             group = IncludeGroup.nonsystem_other
 216             if 'external/' in include.get_included_path():
 217                 path = self._get_path(included_file, group, None)
 218             else:
 219                 path = include.get_included_path()
 220         elif included_file.get_name() == 'gmxpre.h':
 221             group = IncludeGroup.pre
 222             path = self._get_path(included_file, group, None)
 223         elif included_file.get_name() == 'config.h':
 224             group = IncludeGroup.config
 225             path = self._get_path(included_file, group, None)
 226         else:
 227             including_file = include.get_including_file()
 228             group = self._get_gmx_group(including_file, included_file)
 229             path = self._get_path(included_file, group, including_file)
 230         return (group, self._split_path(path), include)
 231
 232     def format_include(self, obj, prev):
 233         """Format an #include directive after sorting."""
 234         result = []
 235         if prev:
 236             if prev[0] != obj[0]:
 237                 # Print empty line between groups
 238                 result.append('\n')
 239             elif prev[1] == obj[1]:
 240                 # Skip duplicates
 241                 return result
 242         include = obj[2]
 243         line = include.get_full_line()
 244         include_re = r'^(?P<head>\s*#\s*include\s+)["<][^">]*[">](?P<tail>.*)$'
 245         match = re.match(include_re, line)
 246         assert match
 247         if include.is_system():
 248             path = '<{0}>'.format(self._join_path(obj[1]))
 249         else:
 250             path = '"{0}"'.format(self._join_path(obj[1]))
 251         result.append('{0}{1}{2}\n'.format(match.group('head'), path, match.group('tail')))
 252         return result
 253
 254 class IncludeSorter(object):
 255
 256     """High-level logic for sorting includes.
 257
 258     This class contains the high-level logic for sorting include statements.
 259     The actual ordering and formatting the includes is delegated to a sort method
 260     (see GroupedSorter) to keep things separated.
 261     """
 262
 263     def __init__(self, sortmethod=None, quiet=True):
 264         """Initialize the include sorter with the given sorter and options."""
 265         if not sortmethod:
 266             sortmethod = GroupedSorter()
 267         self._sortmethod = sortmethod
 268         self._quiet = quiet
 269         self._changed = False
 270
 271     def _sort_include_block(self, block, lines):
 272         """Sort a single include block.
 273
 274         Returns a new list of lines for the block.
 275         If anything is changed, self._changed is set to True, and the caller
 276         can check that."""
 277         includes = map(self._sortmethod.get_sortable_object, block.get_includes())
 278         includes.sort()
 279         result = []
 280         prev = None
 281         current_line_number = block.get_first_line()-1
 282         for include in includes:
 283             newlines = self._sortmethod.format_include(include, prev)
 284             result.extend(newlines)
 285             if not self._changed:
 286                 for offset, newline in enumerate(newlines):
 287                     if lines[current_line_number + offset] != newline:
 288                         self._changed = True
 289                         break
 290                 current_line_number += len(newlines)
 291             prev = include
 292         return result
 293
 294     def sort_includes(self, fileobj):
 295         """Sort all includes in a file."""
 296         lines = fileobj.get_contents()
 297         # Format into a list first:
 298         #  - avoid bugs or issues in the script truncating the file
 299         #  - can check whether anything was changed before touching the file
 300         newlines = []
 301         prev = 0
 302         self._changed = False
 303         for block in fileobj.get_include_blocks():
 304             newlines.extend(lines[prev:block.get_first_line()-1])
 305             newlines.extend(self._sort_include_block(block, lines))
 306             # The returned values are 1-based, but indexing here is 0-based,
 307             # so an explicit +1 is not needed.
 308             prev = block.get_last_line()
 309         if self._changed:
 310             if not self._quiet:
 311                 sys.stderr.write('{0}: includes reformatted\n'.format(fileobj.get_relpath()))
 312             newlines.extend(lines[prev:])
 313             with open(fileobj.get_abspath(), 'w') as fp:
 314                 fp.write(''.join(newlines))
 315
 316     def check_sorted(self, fileobj):
 317         """Check that includes within a file are sorted."""
 318         # TODO: Make the checking work without full contents of the file
 319         lines = fileobj.get_contents()
 320         is_sorted = True
 321         details = None
 322         for block in fileobj.get_include_blocks():
 323             self._changed = False
 324             sorted_lines = self._sort_include_block(block, lines)
 325             if self._changed:
 326                 is_sorted = False
 327                 # TODO: Do a proper diff to show the actual changes.
 328                 if details is None:
 329                     details = ["Correct order/style is:"]
 330                 else:
 331                     details.append("    ...")
 332                 details.extend(["    " + x.rstrip() for x in sorted_lines])
 333         return (is_sorted, details)
 334
 335 def main():
 336     """Run the include sorter script."""
 337     import os
 338     import sys
 339
 340     from optparse import OptionParser
 341
 342     from gmxtree import GromacsTree
 343     from reporter import Reporter
 344
 345     parser = OptionParser()
 346     parser.add_option('-S', '--source-root',
 347                       help='Source tree root directory')
 348     parser.add_option('-B', '--build-root',
 349                       help='Build tree root directory')
 350     parser.add_option('-F', '--files',
 351                       help='Specify files to sort')
 352     parser.add_option('-q', '--quiet', action='store_true',
 353                       help='Do not write status messages')
 354     # This is for evaluating different options; can be removed from the final
 355     # version.
 356     parser.add_option('-s', '--style', type='choice', default='pub-priv',
 357                       choices=('single-group', 'pub-priv', 'pub-local'),
 358                       help='Style for GROMACS includes')
 359     parser.add_option('--absolute', action='store_true',
 360                       help='Write all include paths relative to src/')
 361     options, args = parser.parse_args()
 362
 363     filelist = args
 364     if options.files:
 365         if options.files == '-':
 366             lines = sys.stdin.readlines()
 367         else:
 368             with open(options.files, 'r') as fp:
 369                 lines = fp.readlines()
 370         filelist.extend([x.strip() for x in lines])
 371
 372     reporter = Reporter(quiet=True)
 373
 374     if not options.quiet:
 375         sys.stderr.write('Scanning source tree...\n')
 376     tree = GromacsTree(options.source_root, options.build_root, reporter)
 377     tree.load_installed_file_list()
 378     files = []
 379     for filename in filelist:
 380         fileobj = tree.get_file(os.path.abspath(filename))
 381         if not fileobj:
 382             sys.stderr.write('warning: ignoring unknown file {0}\n'.format(filename))
 383             continue
 384         files.append(fileobj)
 385     if not options.quiet:
 386         sys.stderr.write('Reading source files...\n')
 387     tree.scan_files(only_files=files, keep_contents=True)
 388     extfiles = set(files)
 389     for fileobj in files:
 390         for included_file in fileobj.get_includes():
 391             other_file = included_file.get_file()
 392             if other_file:
 393                 extfiles.add(other_file)
 394     if not options.quiet:
 395         sys.stderr.write('Reading Doxygen XML files...\n')
 396     tree.load_xml(only_files=extfiles)
 397
 398     if not options.quiet:
 399         sys.stderr.write('Sorting includes...\n')
 400
 401     sorter = IncludeSorter(GroupedSorter(options.style, options.absolute), options.quiet)
 402
 403     for fileobj in files:
 404         sorter.sort_includes(fileobj)
 405
 406 if __name__ == '__main__':
 407     main()