bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30
  31
  32 def ignoreRemoval(include, toAdd, absFileName, moduleRules):
  33     # global rules
  34
  35     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  36     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  37         hdl = include.replace(".hpp", ".hdl")
  38         if hdl in toAdd:
  39             return True
  40
  41     # Avoid debug STL.
  42     debugStl = {
  43         "array": ("debug/array", ),
  44         "bitset": ("debug/bitset", ),
  45         "deque": ("debug/deque", ),
  46         "forward_list": ("debug/forward_list", ),
  47         "list": ("debug/list", ),
  48         "map": ("debug/map.h", "debug/multimap.h"),
  49         "set": ("debug/set.h", "debug/multiset.h"),
  50         "unordered_map": ("debug/unordered_map", ),
  51         "unordered_set": ("debug/unordered_set", ),
  52         "vector": ("debug/vector", ),
  53     }
  54     for k, values in debugStl.items():
  55         if include == k:
  56             for value in values:
  57                 if value in toAdd:
  58                     return True
  59
  60     # Avoid proposing to use libstdc++ internal headers.
  61     bits = {
  62         "exception": "bits/exception.h",
  63         "memory": "bits/shared_ptr.h",
  64         "functional": "bits/std_function.h",
  65         "cmath": "bits/std_abs.h",
  66         "ctime": "bits/types/clock_t.h",
  67         "cstdint": "bits/stdint-uintn.h",
  68     }
  69     for k, v in bits.items():
  70         if include == k and v in toAdd:
  71             return True
  72
  73     # Avoid proposing o3tl fw declaration
  74     o3tl = {
  75         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  76         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  77         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  78     }
  79     for k, v, in o3tl.items():
  80         if include == k and v in toAdd:
  81             return True
  82
  83     # Follow boost documentation.
  84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  85         return True
  86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  87         return True
  88     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  89         return True
  90     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  91         return True
  92     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  93         return True
  94
  95     # Avoid .hxx to .h proposals in basic css/uno/* API
  96     unoapi = {
  97         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
  98         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
  99         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 100         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 101     }
 102     for k, v in unoapi.items():
 103         if include == k and v in toAdd:
 104             return True
 105
 106     # 3rd-party, non-self-contained headers.
 107     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 108         return True
 109     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 110         return True
 111     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 112         return True
 113
 114     noRemove = (
 115         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 116         # removing this.
 117         "sal/config.h",
 118         # Works around a build breakage specific to the broken Android
 119         # toolchain.
 120         "android/compatibility.hxx",
 121     )
 122     if include in noRemove:
 123         return True
 124
 125     # Ignore when <foo> is to be replaced with "foo".
 126     if include in toAdd:
 127         return True
 128
 129     fileName = os.path.relpath(absFileName, os.getcwd())
 130
 131     # Skip headers used only for compile test
 132     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 133         if include.endswith(".hpp"):
 134             return True
 135
 136     # yaml rules
 137
 138     if "excludelist" in moduleRules.keys():
 139         excludelistRules = moduleRules["excludelist"]
 140         if fileName in excludelistRules.keys():
 141             if include in excludelistRules[fileName]:
 142                 return True
 143
 144     return False
 145
 146
 147 def unwrapInclude(include):
 148     # Drop <> or "" around the include.
 149     return include[1:-1]
 150
 151
 152 def processIWYUOutput(iwyuOutput, moduleRules, fileName):
 153     inAdd = False
 154     toAdd = []
 155     inRemove = False
 156     toRemove = []
 157     currentFileName = None
 158
 159     for line in iwyuOutput:
 160         line = line.strip()
 161
 162         # Bail out if IWYU gave an error due to non self-containedness
 163         if re.match ("(.*): error: (.*)", line):
 164             return -1
 165
 166         if len(line) == 0:
 167             if inRemove:
 168                 inRemove = False
 169                 continue
 170             if inAdd:
 171                 inAdd = False
 172                 continue
 173
 174         shouldAdd = fileName + " should add these lines:"
 175         match = re.match(shouldAdd, line)
 176         if match:
 177             currentFileName = match.group(0).split(' ')[0]
 178             inAdd = True
 179             continue
 180
 181         shouldRemove = fileName + " should remove these lines:"
 182         match = re.match(shouldRemove, line)
 183         if match:
 184             currentFileName = match.group(0).split(' ')[0]
 185             inRemove = True
 186             continue
 187
 188         if inAdd:
 189             match = re.match('#include ([^ ]+)', line)
 190             if match:
 191                 include = unwrapInclude(match.group(1))
 192                 toAdd.append(include)
 193             else:
 194                 # Forward declaration.
 195                 toAdd.append(line)
 196
 197         if inRemove:
 198             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 199             if match:
 200                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 201                 # indeed unused or they may removed to be replaced with an include. And we want to
 202                 # avoid the later.
 203                 include = unwrapInclude(match.group(1))
 204                 lineno = match.group(2)
 205                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
 206                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 207
 208     for remove in sorted(toRemove):
 209         print("ERROR: %s: remove not needed include" % remove)
 210     return len(toRemove)
 211
 212
 213 def run_tool(task_queue, failed_files):
 214     while True:
 215         invocation, moduleRules = task_queue.get()
 216         if not len(failed_files):
 217             print("[IWYU] " + invocation.split(' ')[-1])
 218             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 219             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
 220             if retcode == -1:
 221                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 222             elif retcode > 0:
 223                 print("ERROR: The following command found unused includes:\n" + invocation)
 224                 failed_files.append(invocation)
 225         task_queue.task_done()
 226
 227
 228 def isInUnoIncludeFile(path):
 229     return path.startswith("include/com/") \
 230             or path.startswith("include/cppu/") \
 231             or path.startswith("include/cppuhelper/") \
 232             or path.startswith("include/osl/") \
 233             or path.startswith("include/rtl/") \
 234             or path.startswith("include/sal/") \
 235             or path.startswith("include/salhelper/") \
 236             or path.startswith("include/systools/") \
 237             or path.startswith("include/typelib/") \
 238             or path.startswith("include/uno/")
 239
 240
 241 def tidy(compileCommands, paths):
 242     return_code = 0
 243     try:
 244         max_task = multiprocessing.cpu_count()
 245         task_queue = queue.Queue(max_task)
 246         failed_files = []
 247         for _ in range(max_task):
 248             t = threading.Thread(target=run_tool, args=(task_queue, failed_files))
 249             t.daemon = True
 250             t.start()
 251
 252         for path in sorted(paths):
 253             if isInUnoIncludeFile(path):
 254                 continue
 255
 256             moduleName = path.split("/")[0]
 257
 258             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 259             moduleRules = {}
 260             if os.path.exists(rulePath):
 261                 moduleRules = yaml.load(open(rulePath))
 262             assume = None
 263             pathAbs = os.path.abspath(path)
 264             compileFile = pathAbs
 265             matches = [i for i in compileCommands if i["file"] == compileFile]
 266             if not len(matches):
 267                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 268                 # code on Linux.
 269                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 270                     assume = moduleRules["assumeFilename"]
 271                 if assume:
 272                     assumeAbs = os.path.abspath(assume)
 273                     compileFile = assumeAbs
 274                     matches = [i for i in compileCommands if i["file"] == compileFile]
 275                     if not len(matches):
 276                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 277                         continue
 278                 else:
 279                     print("WARNING: no compile commands for '" + path + "'")
 280                     continue
 281
 282             _, _, args = matches[0]["command"].partition(" ")
 283             if assume:
 284                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 285
 286             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 287             task_queue.put((invocation, moduleRules))
 288
 289         task_queue.join()
 290         if len(failed_files):
 291             return_code = 1
 292
 293     except KeyboardInterrupt:
 294         print('\nCtrl-C detected, goodbye.')
 295         os.kill(0, 9)
 296
 297     sys.exit(return_code)
 298
 299
 300 def main(argv):
 301     if not len(argv):
 302         print("usage: find-unneeded-includes [FILE]...")
 303         return
 304
 305     try:
 306         with open("compile_commands.json", 'r') as compileCommandsSock:
 307             compileCommands = json.load(compileCommandsSock)
 308     except FileNotFoundError:
 309         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 310         sys.exit(-1)
 311
 312     tidy(compileCommands, paths=argv)
 313
 314 if __name__ == '__main__':
 315     main(sys.argv[1:])
 316
 317 # vim:set shiftwidth=4 softtabstop=4 expandtab: