bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import json
  21 import multiprocessing
  22 import os
  23 import queue
  24 import re
  25 import subprocess
  26 import sys
  27 import threading
  28 import yaml
  29 import argparse
  30 import pathlib
  31
  32
  33 def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
  34     # global rules
  35
  36     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  37     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  38         hdl = include.replace(".hpp", ".hdl")
  39         if hdl in toAdd:
  40             return True
  41
  42     # Avoid debug STL.
  43     debugStl = {
  44         "array": ("debug/array", ),
  45         "bitset": ("debug/bitset", ),
  46         "deque": ("debug/deque", ),
  47         "forward_list": ("debug/forward_list", ),
  48         "list": ("debug/list", ),
  49         "map": ("debug/map.h", "debug/multimap.h"),
  50         "set": ("debug/set.h", "debug/multiset.h"),
  51         "unordered_map": ("debug/unordered_map", ),
  52         "unordered_set": ("debug/unordered_set", ),
  53         "vector": ("debug/vector", ),
  54     }
  55     for k, values in debugStl.items():
  56         if include == k:
  57             for value in values:
  58                 if value in toAdd:
  59                     return True
  60
  61     # Avoid proposing to use libstdc++ internal headers.
  62     bits = {
  63         "exception": "bits/exception.h",
  64         "memory": "bits/shared_ptr.h",
  65         "functional": "bits/std_function.h",
  66         "cmath": "bits/std_abs.h",
  67         "ctime": "bits/types/clock_t.h",
  68         "cstdint": "bits/stdint-uintn.h",
  69     }
  70     for k, v in bits.items():
  71         if include == k and v in toAdd:
  72             return True
  73
  74     # Avoid proposing o3tl fw declaration
  75     o3tl = {
  76         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  77         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  78     }
  79     for k, v, in o3tl.items():
  80         if include == k and v in toAdd:
  81             return True
  82
  83     # Follow boost documentation.
  84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  85         return True
  86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  87         return True
  88     if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
  89         return True
  90     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  91         return True
  92     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  93         return True
  94     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  95         return True
  96
  97     # Avoid .hxx to .h proposals in basic css/uno/* API
  98     unoapi = {
  99         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
 100         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
 101         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 102         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 103     }
 104     for k, v in unoapi.items():
 105         if include == k and v in toAdd:
 106             return True
 107
 108     # 3rd-party, non-self-contained headers.
 109     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 110         return True
 111     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 112         return True
 113     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 114         return True
 115
 116     noRemove = (
 117         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 118         # removing this.
 119         "sal/config.h",
 120         # Works around a build breakage specific to the broken Android
 121         # toolchain.
 122         "android/compatibility.hxx",
 123         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
 124         "osl/endian.h",
 125     )
 126     if include in noRemove:
 127         return True
 128
 129     # Ignore when <foo> is to be replaced with "foo".
 130     if include in toAdd:
 131         return True
 132
 133     fileName = os.path.relpath(absFileName, os.getcwd())
 134
 135     # Skip headers used only for compile test
 136     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 137         if include.endswith(".hpp"):
 138             return True
 139
 140     # yaml rules, except when --noexclude is given
 141
 142     if "excludelist" in moduleRules.keys() and not noexclude:
 143         excludelistRules = moduleRules["excludelist"]
 144         if fileName in excludelistRules.keys():
 145             if include in excludelistRules[fileName]:
 146                 return True
 147
 148     return False
 149
 150
 151 def unwrapInclude(include):
 152     # Drop <> or "" around the include.
 153     return include[1:-1]
 154
 155
 156 def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude):
 157     inAdd = False
 158     toAdd = []
 159     inRemove = False
 160     toRemove = []
 161     currentFileName = None
 162
 163     for line in iwyuOutput:
 164         line = line.strip()
 165
 166         # Bail out if IWYU gave an error due to non self-containedness
 167         if re.match ("(.*): error: (.*)", line):
 168             return -1
 169
 170         if len(line) == 0:
 171             if inRemove:
 172                 inRemove = False
 173                 continue
 174             if inAdd:
 175                 inAdd = False
 176                 continue
 177
 178         shouldAdd = fileName + " should add these lines:"
 179         match = re.match(shouldAdd, line)
 180         if match:
 181             currentFileName = match.group(0).split(' ')[0]
 182             inAdd = True
 183             continue
 184
 185         shouldRemove = fileName + " should remove these lines:"
 186         match = re.match(shouldRemove, line)
 187         if match:
 188             currentFileName = match.group(0).split(' ')[0]
 189             inRemove = True
 190             continue
 191
 192         if inAdd:
 193             match = re.match('#include ([^ ]+)', line)
 194             if match:
 195                 include = unwrapInclude(match.group(1))
 196                 toAdd.append(include)
 197             else:
 198                 # Forward declaration.
 199                 toAdd.append(line)
 200
 201         if inRemove:
 202             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 203             if match:
 204                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 205                 # indeed unused or they may removed to be replaced with an include. And we want to
 206                 # avoid the later.
 207                 include = unwrapInclude(match.group(1))
 208                 lineno = match.group(2)
 209                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
 210                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 211
 212     for remove in sorted(toRemove):
 213         print("ERROR: %s: remove not needed include" % remove)
 214     return len(toRemove)
 215
 216
 217 def run_tool(task_queue, failed_files, dontstop, noexclude):
 218     while True:
 219         invocation, moduleRules = task_queue.get()
 220         if not len(failed_files):
 221             print("[IWYU] " + invocation.split(' ')[-1])
 222             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 223             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude)
 224             if retcode == -1:
 225                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 226             elif retcode > 0:
 227                 print("ERROR: The following command found unused includes:\n" + invocation)
 228                 if not dontstop:
 229                     failed_files.append(invocation)
 230         task_queue.task_done()
 231
 232
 233 def isInUnoIncludeFile(path):
 234     return path.startswith("include/com/") \
 235             or path.startswith("include/cppu/") \
 236             or path.startswith("include/cppuhelper/") \
 237             or path.startswith("include/osl/") \
 238             or path.startswith("include/rtl/") \
 239             or path.startswith("include/sal/") \
 240             or path.startswith("include/salhelper/") \
 241             or path.startswith("include/systools/") \
 242             or path.startswith("include/typelib/") \
 243             or path.startswith("include/uno/")
 244
 245
 246 def tidy(compileCommands, paths, dontstop, noexclude):
 247     return_code = 0
 248
 249     try:
 250         max_task = multiprocessing.cpu_count()
 251         task_queue = queue.Queue(max_task)
 252         failed_files = []
 253         for _ in range(max_task):
 254             t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude))
 255             t.daemon = True
 256             t.start()
 257
 258         for path in sorted(paths):
 259             if isInUnoIncludeFile(path):
 260                 continue
 261
 262             # IWYU fails on these with #error: don't use this in new code
 263             if path.startswith("include/vcl/toolkit"):
 264                 continue
 265
 266             moduleName = path.split("/")[0]
 267
 268             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 269             moduleRules = {}
 270             if os.path.exists(rulePath):
 271                 moduleRules = yaml.full_load(open(rulePath))
 272             assume = None
 273             pathAbs = os.path.abspath(path)
 274             compileFile = pathAbs
 275             matches = [i for i in compileCommands if i["file"] == compileFile]
 276             if not len(matches):
 277                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 278                 # code on Linux.
 279                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 280                     assume = moduleRules["assumeFilename"]
 281                 if assume:
 282                     assumeAbs = os.path.abspath(assume)
 283                     compileFile = assumeAbs
 284                     matches = [i for i in compileCommands if i["file"] == compileFile]
 285                     if not len(matches):
 286                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 287                         continue
 288                 else:
 289                     print("WARNING: no compile commands for '" + path + "'")
 290                     continue
 291
 292             _, _, args = matches[0]["command"].partition(" ")
 293             if assume:
 294                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 295
 296             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 297             task_queue.put((invocation, moduleRules))
 298
 299         task_queue.join()
 300         if len(failed_files):
 301             return_code = 1
 302
 303     except KeyboardInterrupt:
 304         print('\nCtrl-C detected, goodbye.')
 305         os.kill(0, 9)
 306
 307     sys.exit(return_code)
 308
 309
 310 def main(argv):
 311     parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
 312     parser.add_argument('--continue', action='store_true',
 313                     help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
 314     parser.add_argument('Files' , nargs='*',
 315                     help='The files to be checked')
 316     parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
 317                     help='Recursively search a directory for source files to check')
 318     parser.add_argument('--headers', action='store_true',
 319                     help='Check header files. If omitted, check source files. Use with --recursive.')
 320     parser.add_argument('--noexclude', action='store_true',
 321                     help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
 322
 323     args = parser.parse_args()
 324
 325     if not len(argv):
 326         parser.print_help()
 327         return
 328
 329     list_of_files = []
 330     if args.recursive:
 331         for root, dirs, files in os.walk(args.recursive[0]):
 332             for file in files:
 333                 if args.headers:
 334                     if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
 335                         list_of_files.append(os.path.join(root,file))
 336                 else:
 337                     if (file.endswith(".cxx") or file.endswith(".c")):
 338                         list_of_files.append(os.path.join(root,file))
 339     else:
 340         list_of_files = args.Files
 341
 342     try:
 343         with open("compile_commands.json", 'r') as compileCommandsSock:
 344             compileCommands = json.load(compileCommandsSock)
 345     except FileNotFoundError:
 346         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 347         sys.exit(-1)
 348
 349     # quickly sanity check whether files with exceptions in yaml still exists
 350     # only check for the module of the very first filename passed
 351
 352     # Verify there are files selected for checking, with --recursive it
 353     # may happen that there are in fact no C/C++ files in a module directory
 354     if not list_of_files:
 355         print("No files found to check!")
 356         sys.exit(-2)
 357
 358     moduleName = sorted(list_of_files)[0].split("/")[0]
 359     rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 360     moduleRules = {}
 361     if os.path.exists(rulePath):
 362         moduleRules = yaml.full_load(open(rulePath))
 363     if "excludelist" in moduleRules.keys():
 364         excludelistRules = moduleRules["excludelist"]
 365         for pathname in excludelistRules.keys():
 366             file = pathlib.Path(pathname)
 367             if not file.exists():
 368                 print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
 369
 370     tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude)
 371
 372 if __name__ == '__main__':
 373     main(sys.argv[1:])
 374
 375 # vim:set shiftwidth=4 softtabstop=4 expandtab: