bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - blacklist mechanism, so a warning is either fixed or blacklisted
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30
  31
  32 def ignoreRemoval(include, toAdd, absFileName, moduleRules):
  33     # global rules
  34
  35     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  36     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  37         hdl = include.replace(".hpp", ".hdl")
  38         if hdl in toAdd:
  39             return True
  40
  41     # Avoid debug STL.
  42     debugStl = {
  43         "array": ("debug/array", ),
  44         "bitset": ("debug/bitset", ),
  45         "deque": ("debug/deque", ),
  46         "forward_list": ("debug/forward_list", ),
  47         "list": ("debug/list", ),
  48         "map": ("debug/map.h", "debug/multimap.h"),
  49         "set": ("debug/set.h", "debug/multiset.h"),
  50         "unordered_map": ("debug/unordered_map", ),
  51         "unordered_set": ("debug/unordered_set", ),
  52         "vector": ("debug/vector", ),
  53     }
  54     for k, values in debugStl.items():
  55         if include == k:
  56             for value in values:
  57                 if value in toAdd:
  58                     return True
  59
  60     # Avoid proposing to use libstdc++ internal headers.
  61     bits = {
  62         "exception": "bits/exception.h",
  63         "memory": "bits/shared_ptr.h",
  64         "functional": "bits/std_function.h",
  65         "cmath": "bits/std_abs.h",
  66         "ctime": "bits/types/clock_t.h",
  67         "cstdint": "bits/stdint-uintn.h"
  68     }
  69     for k, v in bits.items():
  70         if include == k and v in toAdd:
  71             return True
  72
  73     # Avoid proposing o3tl fw declaration
  74     o3tl = {
  75         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  76         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  77         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  78     }
  79     for k, v, in o3tl.items():
  80         if include == k and v in toAdd:
  81             return True
  82
  83     # Follow boost documentation.
  84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  85         return True
  86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  87         return True
  88     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  89         return True
  90     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  91         return True
  92     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  93         return True
  94
  95     # Avoid .hxx to .h proposals in basic css/uno/* API
  96     unoapi = {
  97         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
  98         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
  99         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 100         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 101     }
 102     for k, v in unoapi.items():
 103         if include == k and v in toAdd:
 104             return True
 105
 106     # 3rd-party, non-self-contained headers.
 107     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 108         return True
 109     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 110         return True
 111
 112     noRemove = (
 113         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 114         # removing this.
 115         "sal/config.h",
 116         # Works around a build breakage specific to the broken Android
 117         # toolchain.
 118         "android/compatibility.hxx",
 119     )
 120     if include in noRemove:
 121         return True
 122
 123     # Ignore when <foo> is to be replaced with "foo".
 124     if include in toAdd:
 125         return True
 126
 127     fileName = os.path.relpath(absFileName, os.getcwd())
 128
 129     # Skip headers used only for compile test
 130     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 131         if include.endswith(".hpp"):
 132             return True
 133
 134     # yaml rules
 135
 136     if "blacklist" in moduleRules.keys():
 137         blacklistRules = moduleRules["blacklist"]
 138         if fileName in blacklistRules.keys():
 139             if include in blacklistRules[fileName]:
 140                 return True
 141
 142     return False
 143
 144
 145 def unwrapInclude(include):
 146     # Drop <> or "" around the include.
 147     return include[1:-1]
 148
 149
 150 def processIWYUOutput(iwyuOutput, moduleRules, fileName):
 151     inAdd = False
 152     toAdd = []
 153     inRemove = False
 154     toRemove = []
 155     currentFileName = None
 156
 157     for line in iwyuOutput:
 158         line = line.strip()
 159
 160         # Bail out if IWYU gave an error due to non self-containedness
 161         if re.match ("(.*): error: (.*)", line):
 162             return -1
 163
 164         if len(line) == 0:
 165             if inRemove:
 166                 inRemove = False
 167                 continue
 168             if inAdd:
 169                 inAdd = False
 170                 continue
 171
 172         shouldAdd = fileName + " should add these lines:"
 173         match = re.match(shouldAdd, line)
 174         if match:
 175             currentFileName = match.group(0).split(' ')[0]
 176             inAdd = True
 177             continue
 178
 179         shouldRemove = fileName + " should remove these lines:"
 180         match = re.match(shouldRemove, line)
 181         if match:
 182             currentFileName = match.group(0).split(' ')[0]
 183             inRemove = True
 184             continue
 185
 186         if inAdd:
 187             match = re.match('#include ([^ ]+)', line)
 188             if match:
 189                 include = unwrapInclude(match.group(1))
 190                 toAdd.append(include)
 191             else:
 192                 # Forward declaration.
 193                 toAdd.append(line)
 194
 195         if inRemove:
 196             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 197             if match:
 198                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 199                 # indeed unused or they may removed to be replaced with an include. And we want to
 200                 # avoid the later.
 201                 include = unwrapInclude(match.group(1))
 202                 lineno = match.group(2)
 203                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
 204                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 205
 206     for remove in sorted(toRemove):
 207         print("ERROR: %s: remove not needed include" % remove)
 208     return len(toRemove)
 209
 210
 211 def run_tool(task_queue, failed_files):
 212     while True:
 213         invocation, moduleRules = task_queue.get()
 214         if not len(failed_files):
 215             print("[IWYU] " + invocation.split(' ')[-1])
 216             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 217             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
 218             if retcode == -1:
 219                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 220             elif retcode > 0:
 221                 print("ERROR: The following command found unused includes:\n" + invocation)
 222                 failed_files.append(invocation)
 223         task_queue.task_done()
 224
 225
 226 def isInUnoIncludeFile(path):
 227     return path.startswith("include/com/") \
 228             or path.startswith("include/cppu/") \
 229             or path.startswith("include/cppuhelper/") \
 230             or path.startswith("include/osl/") \
 231             or path.startswith("include/rtl/") \
 232             or path.startswith("include/sal/") \
 233             or path.startswith("include/salhelper/") \
 234             or path.startswith("include/systools/") \
 235             or path.startswith("include/typelib/") \
 236             or path.startswith("include/uno/")
 237
 238
 239 def tidy(compileCommands, paths):
 240     return_code = 0
 241     try:
 242         max_task = multiprocessing.cpu_count()
 243         task_queue = queue.Queue(max_task)
 244         failed_files = []
 245         for _ in range(max_task):
 246             t = threading.Thread(target=run_tool, args=(task_queue, failed_files))
 247             t.daemon = True
 248             t.start()
 249
 250         for path in sorted(paths):
 251             if isInUnoIncludeFile(path):
 252                 continue
 253
 254             moduleName = path.split("/")[0]
 255
 256             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 257             moduleRules = {}
 258             if os.path.exists(rulePath):
 259                 moduleRules = yaml.load(open(rulePath))
 260             assume = None
 261             pathAbs = os.path.abspath(path)
 262             compileFile = pathAbs
 263             matches = [i for i in compileCommands if i["file"] == compileFile]
 264             if not len(matches):
 265                 if "assumeFilename" in moduleRules.keys():
 266                     assume = moduleRules["assumeFilename"]
 267                 if assume:
 268                     assumeAbs = os.path.abspath(assume)
 269                     compileFile = assumeAbs
 270                     matches = [i for i in compileCommands if i["file"] == compileFile]
 271                     if not len(matches):
 272                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 273                         continue
 274                 else:
 275                     print("WARNING: no compile commands for '" + path + "'")
 276                     continue
 277
 278             _, _, args = matches[0]["command"].partition(" ")
 279             if assume:
 280                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 281
 282             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 283             task_queue.put((invocation, moduleRules))
 284
 285         task_queue.join()
 286         if len(failed_files):
 287             return_code = 1
 288
 289     except KeyboardInterrupt:
 290         print('\nCtrl-C detected, goodbye.')
 291         os.kill(0, 9)
 292
 293     sys.exit(return_code)
 294
 295
 296 def main(argv):
 297     if not len(argv):
 298         print("usage: find-unneeded-includes [FILE]...")
 299         return
 300
 301     try:
 302         with open("compile_commands.json", 'r') as compileCommandsSock:
 303             compileCommands = json.load(compileCommandsSock)
 304     except FileNotFoundError:
 305         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 306         sys.exit(-1)
 307
 308     tidy(compileCommands, paths=argv)
 309
 310 if __name__ == '__main__':
 311     main(sys.argv[1:])
 312
 313 # vim:set shiftwidth=4 softtabstop=4 expandtab: