docs/doxygen/includesorter.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2012,2013,2014,2015,2016,2017, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """Include directive sorter for GROMACS.
  37
  38 This module implements an #include directive sorter for GROMACS C/C++ files.
  39 It allows (in most cases) automatically sorting includes and formatting
  40 the paths to use either relative paths or paths relative to src/.
  41 It groups includes in groups of related headers, sorts the headers
  42 alphabetically within each block, and inserts empty lines in between.
  43 It can be run as a standalone script, in which case it requires an up-to-date
  44 list of installed headers and Doxygen XML documentation to be present in the
  45 build tree.  It can also be imported as a module to be embedded in other
  46 scripts.  In the latter case, the IncludeSorter provides the main interface.
  47
  48 The sorting assumes some conventions (e.g., that system headers are included
  49 with angle brackets instead of quotes).  Generally, these conventions are
  50 checked by the check-source.py script.
  51 """
  52
  53 import os.path
  54 import re
  55 import sys
  56
  57 class IncludeGroup(object):
  58
  59     """Enumeration type for grouping includes."""
  60
  61     def __init__(self, value):
  62         """Initialize a IncludeGroup instance.
  63
  64         IncludeGroup.{main,system_c,...} should be used outside the
  65         class instead of calling the constructor.
  66         """
  67         self._value = value
  68
  69     def __cmp__(self, other):
  70         """Order include groups in the desired order."""
  71         return cmp(self._value, other._value)
  72
  73 # gmxpre.h is always first
  74 IncludeGroup.pre = IncludeGroup(0)
  75 # "main" include file for the source file is next
  76 IncludeGroup.main = IncludeGroup(1)
  77 # config.h is next, if present, to keep its location consistent
  78 IncludeGroup.config = IncludeGroup(2)
  79 # Followed by system headers, with C first and C++ following
  80 IncludeGroup.system_c = IncludeGroup(3)
  81 IncludeGroup.system_c_cpp = IncludeGroup(4)
  82 IncludeGroup.system_cpp = IncludeGroup(5)
  83 # System headers not in standard C/C++ are in a separate block
  84 IncludeGroup.system_other = IncludeGroup(6)
  85 # src/external/ contents that are included with quotes go here
  86 IncludeGroup.nonsystem_other = IncludeGroup(7)
  87 # Other GROMACS headers
  88 IncludeGroup.gmx_general = IncludeGroup(8)
  89 # This group is for shared (unit) testing utilities
  90 IncludeGroup.gmx_test = IncludeGroup(9)
  91 # This group is for headers local to the including file/module
  92 IncludeGroup.gmx_local = IncludeGroup(10)
  93
  94 class GroupedSorter(object):
  95
  96     """Grouping and formatting logic for #include directives.
  97
  98     This class implements the actual logic that decides how includes are
  99     grouped and sorted, and how they are formatted."""
 100
 101     # These variables contain the list of system headers for various blocks
 102     _std_c_headers = ['assert.h', 'ctype.h', 'errno.h', 'float.h',
 103             'inttypes.h', 'limits.h', 'math.h', 'signal.h', 'stdarg.h',
 104             'stddef.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h',
 105             'time.h']
 106     _std_c_cpp_headers = ['c' + x[:-2] for x in _std_c_headers]
 107     _std_cpp_headers = ['algorithm', 'array', 'chrono', 'deque', 'exception', 'fstream',
 108             'functional', 'initializer_list', 'iomanip', 'ios', 'iosfwd',
 109             'iostream', 'istream', 'iterator',
 110             'limits', 'list', 'map', 'memory', 'mutex',
 111             'new', 'numeric', 'ostream', 'random',
 112             'regex', 'set', 'sstream', 'stdexcept', 'streambuf', 'string', 'strstream',
 113             'thread', 'tuple', 'type_traits', 'typeindex', 'typeinfo', 'vector', 'utility']
 114
 115     def __init__(self, style='pub-priv', absolute=False):
 116         """Initialize a sorted with the given style."""
 117         if style == 'single-group':
 118             self._local_group = 'none'
 119         elif style == 'pub-priv':
 120             self._local_group = 'private'
 121         else:
 122             self._local_group = 'local'
 123         if absolute:
 124             self._abspath_main = True
 125             self._abspath_local = True
 126         else:
 127             self._abspath_main = False
 128             self._abspath_local = False
 129
 130     def _get_path(self, included_file, group, including_file):
 131         """Compute include path to use for an #include.
 132
 133         The path is made either absolute (i.e., relative to src/), or
 134         relative to the location of the including file, depending on the group
 135         the file is in.
 136         """
 137         use_abspath = including_file is None or group is None
 138         if not use_abspath:
 139             if group in (IncludeGroup.gmx_general, IncludeGroup.gmx_test):
 140                 use_abspath = True
 141             elif group == IncludeGroup.main and self._abspath_main:
 142                 use_abspath = True
 143             elif group == IncludeGroup.gmx_local and self._abspath_local:
 144                 use_abspath = True
 145         if not use_abspath:
 146             fromdir = os.path.dirname(including_file.get_abspath())
 147             relpath = os.path.relpath(included_file.get_abspath(), fromdir)
 148             if not relpath.startswith('..'):
 149                 return relpath
 150         path = included_file.get_relpath()
 151         assert path.startswith('src/')
 152         return path[4:]
 153
 154     def _get_gmx_group(self, including_file, included_file):
 155         """Determine group for GROMACS headers.
 156
 157         Helper function to determine the group for an #include directive
 158         when the #include is in one of the gmx_* groups (or in the main group).
 159         """
 160         main_header = including_file.get_main_header()
 161         if main_header and main_header == included_file:
 162             return IncludeGroup.main
 163         if included_file.get_directory().get_name() == 'testutils':
 164             return IncludeGroup.gmx_test
 165         if including_file.get_directory().contains(included_file):
 166             if self._local_group == 'local':
 167                 return IncludeGroup.gmx_local
 168             if self._local_group == 'private':
 169                 if included_file.api_type_is_reliable() \
 170                         and included_file.is_module_internal():
 171                     return IncludeGroup.gmx_local
 172                 if not included_file.api_type_is_reliable() \
 173                         and including_file.get_relpath().startswith('src/programs'):
 174                     return IncludeGroup.gmx_local
 175         if included_file.is_test_file():
 176             return IncludeGroup.gmx_test
 177         return IncludeGroup.gmx_general
 178
 179     def _split_path(self, path):
 180         """Split include path into sortable compoments.
 181
 182         Plain string on the full path in the #include directive causes some
 183         unintuitive behavior, so this splits the path into a tuple at
 184         points that allow more natural sorting: primary sort criterion is the
 185         directory name, followed by the basename (without extension) of the
 186         included file.
 187         """
 188         path_components = list(os.path.split(path))
 189         path_components[1] = os.path.splitext(path_components[1])
 190         return tuple(path_components)
 191
 192     def _join_path(self, path_components):
 193         """Reconstruct path from the return value of _split_path."""
 194         return os.path.join(path_components[0], ''.join(path_components[1]))
 195
 196     def get_sortable_object(self, include):
 197         """Produce a sortable, opaque object for an include.
 198
 199         Includes are sorted by calling this function for each #include object,
 200         and sorting the list made up of these objects (using the default
 201         comparison operators).  Each element from the sorted list is then
 202         passed to format_include(), which extracts information from the opaque
 203         object and formats the #include directive for output.
 204         """
 205         included_file = include.get_file()
 206         if not included_file:
 207             path = include.get_included_path()
 208             if path in self._std_c_headers:
 209                 group = IncludeGroup.system_c
 210             elif path in self._std_c_cpp_headers:
 211                 group = IncludeGroup.system_c_cpp
 212             elif path in self._std_cpp_headers:
 213                 group = IncludeGroup.system_cpp
 214             else:
 215                 group = IncludeGroup.system_other
 216         elif included_file.is_external():
 217             group = IncludeGroup.nonsystem_other
 218             if 'external/' in include.get_included_path():
 219                 path = self._get_path(included_file, group, None)
 220             else:
 221                 path = include.get_included_path()
 222         elif included_file.get_name() == 'gmxpre.h':
 223             group = IncludeGroup.pre
 224             path = self._get_path(included_file, group, None)
 225         elif included_file.get_name() == 'config.h':
 226             group = IncludeGroup.config
 227             path = self._get_path(included_file, group, None)
 228         else:
 229             including_file = include.get_including_file()
 230             group = self._get_gmx_group(including_file, included_file)
 231             path = self._get_path(included_file, group, including_file)
 232         return (group, self._split_path(path), include)
 233
 234     def format_include(self, obj, prev):
 235         """Format an #include directive after sorting."""
 236         result = []
 237         if prev:
 238             if prev[0] != obj[0]:
 239                 # Print empty line between groups
 240                 result.append('\n')
 241             elif prev[1] == obj[1]:
 242                 # Skip duplicates
 243                 return result
 244         include = obj[2]
 245         line = include.get_full_line()
 246         include_re = r'^(?P<head>\s*#\s*include\s+)["<][^">]*[">](?P<tail>.*)$'
 247         match = re.match(include_re, line)
 248         assert match
 249         if include.is_system():
 250             path = '<{0}>'.format(self._join_path(obj[1]))
 251         else:
 252             path = '"{0}"'.format(self._join_path(obj[1]))
 253         result.append('{0}{1}{2}\n'.format(match.group('head'), path, match.group('tail')))
 254         return result
 255
 256 class IncludeSorter(object):
 257
 258     """High-level logic for sorting includes.
 259
 260     This class contains the high-level logic for sorting include statements.
 261     The actual ordering and formatting the includes is delegated to a sort method
 262     (see GroupedSorter) to keep things separated.
 263     """
 264
 265     def __init__(self, sortmethod=None, quiet=True):
 266         """Initialize the include sorter with the given sorter and options."""
 267         if not sortmethod:
 268             sortmethod = GroupedSorter()
 269         self._sortmethod = sortmethod
 270         self._quiet = quiet
 271         self._changed = False
 272
 273     def _sort_include_block(self, block, lines):
 274         """Sort a single include block.
 275
 276         Returns a new list of lines for the block.
 277         If anything is changed, self._changed is set to True, and the caller
 278         can check that."""
 279         includes = map(self._sortmethod.get_sortable_object, block.get_includes())
 280         includes.sort()
 281         result = []
 282         prev = None
 283         current_line_number = block.get_first_line()-1
 284         for include in includes:
 285             newlines = self._sortmethod.format_include(include, prev)
 286             result.extend(newlines)
 287             if not self._changed:
 288                 for offset, newline in enumerate(newlines):
 289                     if lines[current_line_number + offset] != newline:
 290                         self._changed = True
 291                         break
 292                 current_line_number += len(newlines)
 293             prev = include
 294         return result
 295
 296     def sort_includes(self, fileobj):
 297         """Sort all includes in a file."""
 298         lines = fileobj.get_contents()
 299         # Format into a list first:
 300         #  - avoid bugs or issues in the script truncating the file
 301         #  - can check whether anything was changed before touching the file
 302         newlines = []
 303         prev = 0
 304         self._changed = False
 305         for block in fileobj.get_include_blocks():
 306             newlines.extend(lines[prev:block.get_first_line()-1])
 307             newlines.extend(self._sort_include_block(block, lines))
 308             # The returned values are 1-based, but indexing here is 0-based,
 309             # so an explicit +1 is not needed.
 310             prev = block.get_last_line()
 311         if self._changed:
 312             if not self._quiet:
 313                 sys.stderr.write('{0}: includes reformatted\n'.format(fileobj.get_relpath()))
 314             newlines.extend(lines[prev:])
 315             with open(fileobj.get_abspath(), 'w') as fp:
 316                 fp.write(''.join(newlines))
 317
 318     def check_sorted(self, fileobj):
 319         """Check that includes within a file are sorted."""
 320         # TODO: Make the checking work without full contents of the file
 321         lines = fileobj.get_contents()
 322         is_sorted = True
 323         details = None
 324         for block in fileobj.get_include_blocks():
 325             self._changed = False
 326             sorted_lines = self._sort_include_block(block, lines)
 327             if self._changed:
 328                 is_sorted = False
 329                 # TODO: Do a proper diff to show the actual changes.
 330                 if details is None:
 331                     details = ["Correct order/style is:"]
 332                 else:
 333                     details.append("    ...")
 334                 details.extend(["    " + x.rstrip() for x in sorted_lines])
 335         return (is_sorted, details)
 336
 337 def main():
 338     """Run the include sorter script."""
 339     import os
 340     import sys
 341
 342     from optparse import OptionParser
 343
 344     from gmxtree import GromacsTree
 345     from reporter import Reporter
 346
 347     parser = OptionParser()
 348     parser.add_option('-S', '--source-root',
 349                       help='Source tree root directory')
 350     parser.add_option('-B', '--build-root',
 351                       help='Build tree root directory')
 352     parser.add_option('-F', '--files',
 353                       help='Specify files to sort')
 354     parser.add_option('-q', '--quiet', action='store_true',
 355                       help='Do not write status messages')
 356     # This is for evaluating different options; can be removed from the final
 357     # version.
 358     parser.add_option('-s', '--style', type='choice', default='pub-priv',
 359                       choices=('single-group', 'pub-priv', 'pub-local'),
 360                       help='Style for GROMACS includes')
 361     parser.add_option('--absolute', action='store_true',
 362                       help='Write all include paths relative to src/')
 363     options, args = parser.parse_args()
 364
 365     filelist = args
 366     if options.files:
 367         if options.files == '-':
 368             lines = sys.stdin.readlines()
 369         else:
 370             with open(options.files, 'r') as fp:
 371                 lines = fp.readlines()
 372         filelist.extend([x.strip() for x in lines])
 373
 374     reporter = Reporter(quiet=True)
 375
 376     if not options.quiet:
 377         sys.stderr.write('Scanning source tree...\n')
 378     if not options.source_root:
 379         sys.stderr.write('Source root required not specified.\n')
 380         sys.exit(2)
 381     if not options.build_root:
 382         sys.stderr.write('Build root required not specified.\n')
 383         sys.exit(2)
 384     tree = GromacsTree(options.source_root, options.build_root, reporter)
 385     tree.load_installed_file_list()
 386     files = []
 387     for filename in filelist:
 388         fileobj = tree.get_file(os.path.abspath(filename))
 389         if not fileobj:
 390             sys.stderr.write('warning: ignoring unknown file {0}\n'.format(filename))
 391             continue
 392         files.append(fileobj)
 393     if not options.quiet:
 394         sys.stderr.write('Reading source files...\n')
 395     tree.scan_files(only_files=files, keep_contents=True)
 396     extfiles = set(files)
 397     for fileobj in files:
 398         for included_file in fileobj.get_includes():
 399             other_file = included_file.get_file()
 400             if other_file:
 401                 extfiles.add(other_file)
 402     if not options.quiet:
 403         sys.stderr.write('Reading Doxygen XML files...\n')
 404     tree.load_xml(only_files=extfiles)
 405
 406     if not options.quiet:
 407         sys.stderr.write('Sorting includes...\n')
 408
 409     sorter = IncludeSorter(GroupedSorter(options.style, options.absolute), options.quiet)
 410
 411     for fileobj in files:
 412         sorter.sort_includes(fileobj)
 413
 414 if __name__ == '__main__':
 415     main()