bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30 import argparse
  31 import pathlib
  32
  33
  34 def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
  35     # global rules
  36
  37     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  38     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  39         hdl = include.replace(".hpp", ".hdl")
  40         if hdl in toAdd:
  41             return True
  42
  43     # Avoid debug STL.
  44     debugStl = {
  45         "array": ("debug/array", ),
  46         "bitset": ("debug/bitset", ),
  47         "deque": ("debug/deque", ),
  48         "forward_list": ("debug/forward_list", ),
  49         "list": ("debug/list", ),
  50         "map": ("debug/map.h", "debug/multimap.h"),
  51         "set": ("debug/set.h", "debug/multiset.h"),
  52         "unordered_map": ("debug/unordered_map", ),
  53         "unordered_set": ("debug/unordered_set", ),
  54         "vector": ("debug/vector", ),
  55     }
  56     for k, values in debugStl.items():
  57         if include == k:
  58             for value in values:
  59                 if value in toAdd:
  60                     return True
  61
  62     # Avoid proposing to use libstdc++ internal headers.
  63     bits = {
  64         "exception": "bits/exception.h",
  65         "memory": "bits/shared_ptr.h",
  66         "functional": "bits/std_function.h",
  67         "cmath": "bits/std_abs.h",
  68         "ctime": "bits/types/clock_t.h",
  69         "cstdint": "bits/stdint-uintn.h",
  70     }
  71     for k, v in bits.items():
  72         if include == k and v in toAdd:
  73             return True
  74
  75     # Avoid proposing o3tl fw declaration
  76     o3tl = {
  77         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  78         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  79     }
  80     for k, v, in o3tl.items():
  81         if include == k and v in toAdd:
  82             return True
  83
  84     # Follow boost documentation.
  85     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  86         return True
  87     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  88         return True
  89     if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
  90         return True
  91     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  92         return True
  93     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  94         return True
  95     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  96         return True
  97
  98     # Avoid .hxx to .h proposals in basic css/uno/* API
  99     unoapi = {
 100         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
 101         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
 102         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 103         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 104     }
 105     for k, v in unoapi.items():
 106         if include == k and v in toAdd:
 107             return True
 108
 109     # 3rd-party, non-self-contained headers.
 110     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 111         return True
 112     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 113         return True
 114     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 115         return True
 116
 117     noRemove = (
 118         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 119         # removing this.
 120         "sal/config.h",
 121         # Works around a build breakage specific to the broken Android
 122         # toolchain.
 123         "android/compatibility.hxx",
 124         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
 125         "osl/endian.h",
 126     )
 127     if include in noRemove:
 128         return True
 129
 130     # Ignore when <foo> is to be replaced with "foo".
 131     if include in toAdd:
 132         return True
 133
 134     fileName = os.path.relpath(absFileName, os.getcwd())
 135
 136     # Skip headers used only for compile test
 137     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 138         if include.endswith(".hpp"):
 139             return True
 140
 141     # yaml rules, except when --noexclude is given
 142
 143     if "excludelist" in moduleRules.keys() and not noexclude:
 144         excludelistRules = moduleRules["excludelist"]
 145         if fileName in excludelistRules.keys():
 146             if include in excludelistRules[fileName]:
 147                 return True
 148
 149     return False
 150
 151
 152 def unwrapInclude(include):
 153     # Drop <> or "" around the include.
 154     return include[1:-1]
 155
 156
 157 def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude):
 158     inAdd = False
 159     toAdd = []
 160     inRemove = False
 161     toRemove = []
 162     currentFileName = None
 163
 164     for line in iwyuOutput:
 165         line = line.strip()
 166
 167         # Bail out if IWYU gave an error due to non self-containedness
 168         if re.match ("(.*): error: (.*)", line):
 169             return -1
 170
 171         if len(line) == 0:
 172             if inRemove:
 173                 inRemove = False
 174                 continue
 175             if inAdd:
 176                 inAdd = False
 177                 continue
 178
 179         shouldAdd = fileName + " should add these lines:"
 180         match = re.match(shouldAdd, line)
 181         if match:
 182             currentFileName = match.group(0).split(' ')[0]
 183             inAdd = True
 184             continue
 185
 186         shouldRemove = fileName + " should remove these lines:"
 187         match = re.match(shouldRemove, line)
 188         if match:
 189             currentFileName = match.group(0).split(' ')[0]
 190             inRemove = True
 191             continue
 192
 193         if inAdd:
 194             match = re.match('#include ([^ ]+)', line)
 195             if match:
 196                 include = unwrapInclude(match.group(1))
 197                 toAdd.append(include)
 198             else:
 199                 # Forward declaration.
 200                 toAdd.append(line)
 201
 202         if inRemove:
 203             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 204             if match:
 205                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 206                 # indeed unused or they may removed to be replaced with an include. And we want to
 207                 # avoid the later.
 208                 include = unwrapInclude(match.group(1))
 209                 lineno = match.group(2)
 210                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
 211                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 212
 213     for remove in sorted(toRemove):
 214         print("ERROR: %s: remove not needed include" % remove)
 215     return len(toRemove)
 216
 217
 218 def run_tool(task_queue, failed_files, dontstop, noexclude):
 219     while True:
 220         invocation, moduleRules = task_queue.get()
 221         if not len(failed_files):
 222             print("[IWYU] " + invocation.split(' ')[-1])
 223             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 224             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude)
 225             if retcode == -1:
 226                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 227             elif retcode > 0:
 228                 print("ERROR: The following command found unused includes:\n" + invocation)
 229                 if not dontstop:
 230                     failed_files.append(invocation)
 231         task_queue.task_done()
 232
 233
 234 def isInUnoIncludeFile(path):
 235     return path.startswith("include/com/") \
 236             or path.startswith("include/cppu/") \
 237             or path.startswith("include/cppuhelper/") \
 238             or path.startswith("include/osl/") \
 239             or path.startswith("include/rtl/") \
 240             or path.startswith("include/sal/") \
 241             or path.startswith("include/salhelper/") \
 242             or path.startswith("include/systools/") \
 243             or path.startswith("include/typelib/") \
 244             or path.startswith("include/uno/")
 245
 246
 247 def tidy(compileCommands, paths, dontstop, noexclude):
 248     return_code = 0
 249
 250     try:
 251         max_task = multiprocessing.cpu_count()
 252         task_queue = queue.Queue(max_task)
 253         failed_files = []
 254         for _ in range(max_task):
 255             t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude))
 256             t.daemon = True
 257             t.start()
 258
 259         for path in sorted(paths):
 260             if isInUnoIncludeFile(path):
 261                 continue
 262
 263             # IWYU fails on these with #error: don't use this in new code
 264             if path.startswith("include/vcl/toolkit"):
 265                 continue
 266
 267             moduleName = path.split("/")[0]
 268
 269             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 270             moduleRules = {}
 271             if os.path.exists(rulePath):
 272                 moduleRules = yaml.full_load(open(rulePath))
 273             assume = None
 274             pathAbs = os.path.abspath(path)
 275             compileFile = pathAbs
 276             matches = [i for i in compileCommands if i["file"] == compileFile]
 277             if not len(matches):
 278                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 279                 # code on Linux.
 280                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 281                     assume = moduleRules["assumeFilename"]
 282                 if assume:
 283                     assumeAbs = os.path.abspath(assume)
 284                     compileFile = assumeAbs
 285                     matches = [i for i in compileCommands if i["file"] == compileFile]
 286                     if not len(matches):
 287                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 288                         continue
 289                 else:
 290                     print("WARNING: no compile commands for '" + path + "'")
 291                     continue
 292
 293             _, _, args = matches[0]["command"].partition(" ")
 294             if assume:
 295                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 296
 297             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 298             task_queue.put((invocation, moduleRules))
 299
 300         task_queue.join()
 301         if len(failed_files):
 302             return_code = 1
 303
 304     except KeyboardInterrupt:
 305         print('\nCtrl-C detected, goodbye.')
 306         os.kill(0, 9)
 307
 308     sys.exit(return_code)
 309
 310
 311 def main(argv):
 312     parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
 313     parser.add_argument('--continue', action='store_true',
 314                     help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
 315     parser.add_argument('Files' , nargs='*',
 316                     help='The files to be checked')
 317     parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
 318                     help='Recursively search a directory for source files to check')
 319     parser.add_argument('--headers', action='store_true',
 320                     help='Check header files. If omitted, check source files. Use with --recursive.')
 321     parser.add_argument('--noexclude', action='store_true',
 322                     help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
 323
 324     args = parser.parse_args()
 325
 326     if not len(argv):
 327         parser.print_help()
 328         return
 329
 330     list_of_files = []
 331     if args.recursive:
 332         for root, dirs, files in os.walk(args.recursive[0]):
 333             for file in files:
 334                 if args.headers:
 335                     if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
 336                         list_of_files.append(os.path.join(root,file))
 337                 else:
 338                     if (file.endswith(".cxx") or file.endswith(".c")):
 339                         list_of_files.append(os.path.join(root,file))
 340     else:
 341         list_of_files = args.Files
 342
 343     try:
 344         with open("compile_commands.json", 'r') as compileCommandsSock:
 345             compileCommands = json.load(compileCommandsSock)
 346     except FileNotFoundError:
 347         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 348         sys.exit(-1)
 349
 350     # quickly sanity check whether files with exceptions in yaml still exists
 351     # only check for the module of the very first filename passed
 352
 353     # Verify there are files selected for checking, with --recursive it
 354     # may happen that there are in fact no C/C++ files in a module directory
 355     if not list_of_files:
 356         print("No files found to check!")
 357         sys.exit(-2)
 358
 359     moduleName = sorted(list_of_files)[0].split("/")[0]
 360     rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 361     moduleRules = {}
 362     if os.path.exists(rulePath):
 363         moduleRules = yaml.full_load(open(rulePath))
 364     if "excludelist" in moduleRules.keys():
 365         excludelistRules = moduleRules["excludelist"]
 366         for pathname in excludelistRules.keys():
 367             file = pathlib.Path(pathname)
 368             if not file.exists():
 369                 print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
 370
 371     tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude)
 372
 373 if __name__ == '__main__':
 374     main(sys.argv[1:])
 375
 376 # vim:set shiftwidth=4 softtabstop=4 expandtab: