bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30
  31
  32 def ignoreRemoval(include, toAdd, absFileName, moduleRules):
  33     # global rules
  34
  35     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  36     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  37         hdl = include.replace(".hpp", ".hdl")
  38         if hdl in toAdd:
  39             return True
  40
  41     # Avoid debug STL.
  42     debugStl = {
  43         "array": ("debug/array", ),
  44         "bitset": ("debug/bitset", ),
  45         "deque": ("debug/deque", ),
  46         "forward_list": ("debug/forward_list", ),
  47         "list": ("debug/list", ),
  48         "map": ("debug/map.h", "debug/multimap.h"),
  49         "set": ("debug/set.h", "debug/multiset.h"),
  50         "unordered_map": ("debug/unordered_map", ),
  51         "unordered_set": ("debug/unordered_set", ),
  52         "vector": ("debug/vector", ),
  53     }
  54     for k, values in debugStl.items():
  55         if include == k:
  56             for value in values:
  57                 if value in toAdd:
  58                     return True
  59
  60     # Avoid proposing to use libstdc++ internal headers.
  61     bits = {
  62         "exception": "bits/exception.h",
  63         "memory": "bits/shared_ptr.h",
  64         "functional": "bits/std_function.h",
  65         "cmath": "bits/std_abs.h",
  66         "ctime": "bits/types/clock_t.h",
  67         "cstdint": "bits/stdint-uintn.h",
  68     }
  69     for k, v in bits.items():
  70         if include == k and v in toAdd:
  71             return True
  72
  73     # Avoid proposing o3tl fw declaration
  74     o3tl = {
  75         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  76         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  77         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  78     }
  79     for k, v, in o3tl.items():
  80         if include == k and v in toAdd:
  81             return True
  82
  83     # Follow boost documentation.
  84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  85         return True
  86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  87         return True
  88     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  89         return True
  90     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  91         return True
  92     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  93         return True
  94
  95     # Avoid .hxx to .h proposals in basic css/uno/* API
  96     unoapi = {
  97         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
  98         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
  99         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 100         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 101     }
 102     for k, v in unoapi.items():
 103         if include == k and v in toAdd:
 104             return True
 105
 106     # 3rd-party, non-self-contained headers.
 107     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 108         return True
 109     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 110         return True
 111     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 112         return True
 113
 114     noRemove = (
 115         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 116         # removing this.
 117         "sal/config.h",
 118         # Works around a build breakage specific to the broken Android
 119         # toolchain.
 120         "android/compatibility.hxx",
 121         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
 122         "osl/endian.h",
 123     )
 124     if include in noRemove:
 125         return True
 126
 127     # Ignore when <foo> is to be replaced with "foo".
 128     if include in toAdd:
 129         return True
 130
 131     fileName = os.path.relpath(absFileName, os.getcwd())
 132
 133     # Skip headers used only for compile test
 134     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 135         if include.endswith(".hpp"):
 136             return True
 137
 138     # yaml rules
 139
 140     if "excludelist" in moduleRules.keys():
 141         excludelistRules = moduleRules["excludelist"]
 142         if fileName in excludelistRules.keys():
 143             if include in excludelistRules[fileName]:
 144                 return True
 145
 146     return False
 147
 148
 149 def unwrapInclude(include):
 150     # Drop <> or "" around the include.
 151     return include[1:-1]
 152
 153
 154 def processIWYUOutput(iwyuOutput, moduleRules, fileName):
 155     inAdd = False
 156     toAdd = []
 157     inRemove = False
 158     toRemove = []
 159     currentFileName = None
 160
 161     for line in iwyuOutput:
 162         line = line.strip()
 163
 164         # Bail out if IWYU gave an error due to non self-containedness
 165         if re.match ("(.*): error: (.*)", line):
 166             return -1
 167
 168         if len(line) == 0:
 169             if inRemove:
 170                 inRemove = False
 171                 continue
 172             if inAdd:
 173                 inAdd = False
 174                 continue
 175
 176         shouldAdd = fileName + " should add these lines:"
 177         match = re.match(shouldAdd, line)
 178         if match:
 179             currentFileName = match.group(0).split(' ')[0]
 180             inAdd = True
 181             continue
 182
 183         shouldRemove = fileName + " should remove these lines:"
 184         match = re.match(shouldRemove, line)
 185         if match:
 186             currentFileName = match.group(0).split(' ')[0]
 187             inRemove = True
 188             continue
 189
 190         if inAdd:
 191             match = re.match('#include ([^ ]+)', line)
 192             if match:
 193                 include = unwrapInclude(match.group(1))
 194                 toAdd.append(include)
 195             else:
 196                 # Forward declaration.
 197                 toAdd.append(line)
 198
 199         if inRemove:
 200             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 201             if match:
 202                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 203                 # indeed unused or they may removed to be replaced with an include. And we want to
 204                 # avoid the later.
 205                 include = unwrapInclude(match.group(1))
 206                 lineno = match.group(2)
 207                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
 208                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 209
 210     for remove in sorted(toRemove):
 211         print("ERROR: %s: remove not needed include" % remove)
 212     return len(toRemove)
 213
 214
 215 def run_tool(task_queue, failed_files):
 216     while True:
 217         invocation, moduleRules = task_queue.get()
 218         if not len(failed_files):
 219             print("[IWYU] " + invocation.split(' ')[-1])
 220             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 221             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
 222             if retcode == -1:
 223                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 224             elif retcode > 0:
 225                 print("ERROR: The following command found unused includes:\n" + invocation)
 226                 failed_files.append(invocation)
 227         task_queue.task_done()
 228
 229
 230 def isInUnoIncludeFile(path):
 231     return path.startswith("include/com/") \
 232             or path.startswith("include/cppu/") \
 233             or path.startswith("include/cppuhelper/") \
 234             or path.startswith("include/osl/") \
 235             or path.startswith("include/rtl/") \
 236             or path.startswith("include/sal/") \
 237             or path.startswith("include/salhelper/") \
 238             or path.startswith("include/systools/") \
 239             or path.startswith("include/typelib/") \
 240             or path.startswith("include/uno/")
 241
 242
 243 def tidy(compileCommands, paths):
 244     return_code = 0
 245     try:
 246         max_task = multiprocessing.cpu_count()
 247         task_queue = queue.Queue(max_task)
 248         failed_files = []
 249         for _ in range(max_task):
 250             t = threading.Thread(target=run_tool, args=(task_queue, failed_files))
 251             t.daemon = True
 252             t.start()
 253
 254         for path in sorted(paths):
 255             if isInUnoIncludeFile(path):
 256                 continue
 257
 258             moduleName = path.split("/")[0]
 259
 260             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 261             moduleRules = {}
 262             if os.path.exists(rulePath):
 263                 moduleRules = yaml.load(open(rulePath))
 264             assume = None
 265             pathAbs = os.path.abspath(path)
 266             compileFile = pathAbs
 267             matches = [i for i in compileCommands if i["file"] == compileFile]
 268             if not len(matches):
 269                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 270                 # code on Linux.
 271                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 272                     assume = moduleRules["assumeFilename"]
 273                 if assume:
 274                     assumeAbs = os.path.abspath(assume)
 275                     compileFile = assumeAbs
 276                     matches = [i for i in compileCommands if i["file"] == compileFile]
 277                     if not len(matches):
 278                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 279                         continue
 280                 else:
 281                     print("WARNING: no compile commands for '" + path + "'")
 282                     continue
 283
 284             _, _, args = matches[0]["command"].partition(" ")
 285             if assume:
 286                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 287
 288             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 289             task_queue.put((invocation, moduleRules))
 290
 291         task_queue.join()
 292         if len(failed_files):
 293             return_code = 1
 294
 295     except KeyboardInterrupt:
 296         print('\nCtrl-C detected, goodbye.')
 297         os.kill(0, 9)
 298
 299     sys.exit(return_code)
 300
 301
 302 def main(argv):
 303     if not len(argv):
 304         print("usage: find-unneeded-includes [FILE]...")
 305         return
 306
 307     try:
 308         with open("compile_commands.json", 'r') as compileCommandsSock:
 309             compileCommands = json.load(compileCommandsSock)
 310     except FileNotFoundError:
 311         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 312         sys.exit(-1)
 313
 314     tidy(compileCommands, paths=argv)
 315
 316 if __name__ == '__main__':
 317     main(sys.argv[1:])
 318
 319 # vim:set shiftwidth=4 softtabstop=4 expandtab: