3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 # This parses the output of 'include-what-you-use', focusing on just removing
8 # not needed includes and providing a relatively conservative output by
9 # filtering out a number of LibreOffice-specific false positives.
11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
12 # you can generate one with 'make vim-ide-integration'.
15 # - excludelist mechanism, so a warning is either fixed or excluded
16 # - works in a plugins-enabled clang build
17 # - no custom configure options required
18 # - no need to generate a dummy library to build a header
21 import multiprocessing
33 def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
36 # Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces.
37 if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
38 hdl = include.replace(".hpp", ".hdl")
44 "array": ("debug/array", ),
45 "bitset": ("debug/bitset", ),
46 "deque": ("debug/deque", ),
47 "forward_list": ("debug/forward_list", ),
48 "list": ("debug/list", ),
49 "map": ("debug/map.h", "debug/multimap.h"),
50 "set": ("debug/set.h", "debug/multiset.h"),
51 "unordered_map": ("debug/unordered_map", ),
52 "unordered_set": ("debug/unordered_set", ),
53 "vector": ("debug/vector", ),
55 for k, values in debugStl.items():
61 # Avoid proposing to use libstdc++ internal headers.
63 "exception": "bits/exception.h",
64 "memory": "bits/shared_ptr.h",
65 "functional": "bits/std_function.h",
66 "cmath": "bits/std_abs.h",
67 "ctime": "bits/types/clock_t.h",
68 "cstdint": "bits/stdint-uintn.h",
70 for k, v in bits.items():
71 if include == k and v in toAdd:
74 # Avoid proposing o3tl fw declaration
76 "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
77 "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
79 for k, v, in o3tl.items():
80 if include == k and v in toAdd:
83 # Follow boost documentation.
84 if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
86 if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
88 if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
90 if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
92 if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
94 if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
97 # Avoid .hxx to .h proposals in basic css/uno/* API
99 "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
100 "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
101 "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
102 "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
104 for k, v in unoapi.items():
105 if include == k and v in toAdd:
108 # 3rd-party, non-self-contained headers.
109 if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
111 if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
113 if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
117 # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
120 # Works around a build breakage specific to the broken Android
122 "android/compatibility.hxx",
123 # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
126 if include in noRemove:
129 # Ignore when <foo> is to be replaced with "foo".
133 fileName = os.path.relpath(absFileName, os.getcwd())
135 # Skip headers used only for compile test
136 if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
137 if include.endswith(".hpp"):
140 # yaml rules, except when --noexclude is given
142 if "excludelist" in moduleRules.keys() and not noexclude:
143 excludelistRules = moduleRules["excludelist"]
144 if fileName in excludelistRules.keys():
145 if include in excludelistRules[fileName]:
151 def unwrapInclude(include):
152 # Drop <> or "" around the include.
156 def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces, finderrors, removefwdd):
162 currentFileName = None
164 for line in iwyuOutput:
167 # Bail out if IWYU gave an error due to non self-containedness
168 if re.match ("(.*): error: (.*)", line):
171 # Bail out if we are in finderrors mode
186 shouldAdd = fileName + " should add these lines:"
187 match = re.match(shouldAdd, line)
189 currentFileName = match.group(0).split(' ')[0]
193 shouldRemove = fileName + " should remove these lines:"
194 match = re.match(shouldRemove, line)
196 currentFileName = match.group(0).split(' ')[0]
201 match = re.match("The full include-list for " + fileName, line)
207 match = re.match('#include ([^ ]+)', line)
209 include = unwrapInclude(match.group(1))
210 toAdd.append(include)
212 # Forward declaration.
215 if inRemove and not checknamespaces:
217 match = re.match("- #include (.*) // lines (.*)-.*", line)
219 include = unwrapInclude(match.group(1))
220 lineno = match.group(2)
221 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
222 toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
225 # Search for obsolete forward declarations, but not header -> fwdecl replacements
226 match = re.match("- (.*;(?: })*)* // lines (.*)-.*", line)
228 fwdDecl = match.group(1)
229 if fwdDecl.endswith(";"):
230 # Remove trailing semicolon.
231 fwdDecl = fwdDecl[:-1]
232 lineno = match.group(2)
233 if not ignoreRemoval(fwdDecl, toAdd, currentFileName, moduleRules, noexclude):
234 toRemove.append("%s:%s: %s" % (currentFileName, lineno, fwdDecl))
239 # match for all possible URE/UNO namespaces, created with:
240 # find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr '\012' '|'
241 # find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr '\012' '|'
242 # and ooo::vba namespaces
243 # plus a few popular ones about other modules
248 'bridge|oleautomation|'
270 'configuration|bootstrap|backend|xml|'
272 'datatransfer|clipboard|dnd|'
273 'deployment|test|ui|'
277 'form|binding|runtime|control|inspection|submission|component|validation|'
293 'packages|zip|manifest|'
294 'presentation|textfield|'
298 'report|inspection|meta|'
301 'script|vba|browse|provider|'
302 'sdb|application|tools|'
314 'text|textfield|docinfo|fieldmaster|'
321 'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|'
323 # ooo::vba and its namespaces
324 'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|'
325 # use of module namespaces, as spotted in the code
326 'analysis|pricing' # sca internals
327 'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals
329 'boost|posix_time|gregorian'
335 'cpp|java|' # for codemaker::
337 'dbaccess|dbahsql|dbaui|dbtools|'
339 'drawinglayer|attribute|geometry|primitive2d|processor2d|'
345 'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal
347 'internal|ColorComponentTag|' # for slideshow internals
353 'mtv|' # for mdds::mtv
354 'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal
356 'odfflatxml|' # filter internal
357 'oox|core|drawingml|ole|vml|'
368 'sax|' # for xml::sax
370 'SchXMLTools|' # for xmloff
371 'sd|slidesorter|cache|controller|model|view|'
374 'sidebar|' # for sfx2::sidebar
376 'star|' # for com::sun::star
377 'std|chrono_literals|literals|'
383 'svx|sdr|contact|table|'
384 'sw|access|annotation|mark|types|util|'
390 'util|db|qe|' # for xmlsearch::
395 'xmloff|token|EnhancedCustomShapeToken' # for xmloff::
400 reason = re.match(ns, line)
402 # Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used
403 # otherwise the used classes name would show up after the '// for'
404 # Cleaning out the respective header (if there is any
405 # - which is not always the case) is for the next run!
406 nameSpace = reason.group(1).split(' ')[0]
407 print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace)
409 # Get the row number, normal IWYU output does not contain this info
410 subprocess.run(["git", "grep", "-n", "namespace.*[^a-zA-Z]"+nameSpace+" *;", fileName])
413 for remove in sorted(toRemove, key=lambda x: int(x.split(":")[1])):
414 print("ERROR: %s: remove not needed forward declaration" % remove)
416 for remove in sorted(toRemove, key=lambda x: int(x.split(":")[1])):
417 print("ERROR: %s: remove not needed include" % remove)
421 def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces, finderrors, removefwdd):
423 invocation, moduleRules = task_queue.get()
424 if not len(failed_files):
425 print("[IWYU] " + invocation.split(' ')[-1])
426 p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
427 retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces, finderrors, removefwdd)
429 if p.returncode == 1:
430 print("Running the IWYU process returned error code:\n" + invocation)
431 if retcode == -1 and not checknamespaces and not removefwdd:
432 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
435 print("ERROR: The following command found unused includes:\n" + invocation)
437 print("ERROR: The following command found unused forward declarations:\n" + invocation)
439 failed_files.append(invocation)
440 task_queue.task_done()
442 # Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished
443 os.system('stty sane')
446 def isInUnoIncludeFile(path):
447 return path.startswith("include/com/") \
448 or path.startswith("include/cppu/") \
449 or path.startswith("include/cppuhelper/") \
450 or path.startswith("include/osl/") \
451 or path.startswith("include/rtl/") \
452 or path.startswith("include/sal/") \
453 or path.startswith("include/salhelper/") \
454 or path.startswith("include/systools/") \
455 or path.startswith("include/typelib/") \
456 or path.startswith("include/uno/")
459 def tidy(compileCommands, paths, dontstop, noexclude, checknamespaces, finderrors, removefwdd):
463 max_task = multiprocessing.cpu_count()
464 task_queue = queue.Queue(max_task)
466 for _ in range(max_task):
467 t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude, checknamespaces, finderrors, removefwdd))
471 for path in sorted(paths):
472 if isInUnoIncludeFile(path):
475 # IWYU fails on these with #error: don't use this in new code
476 if path.startswith("include/vcl/toolkit"):
479 moduleName = path.split("/")[0]
481 rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
483 if os.path.exists(rulePath):
484 moduleRules = yaml.full_load(open(rulePath))
486 pathAbs = os.path.abspath(path)
487 compileFile = pathAbs
488 matches = [i for i in compileCommands if i["file"] == compileFile]
490 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
492 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
493 assume = moduleRules["assumeFilename"]
495 assumeAbs = os.path.abspath(assume)
496 compileFile = assumeAbs
497 matches = [i for i in compileCommands if i["file"] == compileFile]
499 print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
502 print("WARNING: no compile commands for '" + path + "'")
505 _, _, args = matches[0]["command"].partition(" ")
507 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
510 invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
511 # In --fwdecl mode we ask for fw declaration removal suggestions.
512 # In this mode obsolete fw declarations are suggested for removal.
513 # Later we ignore the header removal suggestions, which may be
514 # there because of possibility of replacement with fw declarations
515 # but those and our CI are not reliable enough yet for use
517 invocation = "include-what-you-use -Xiwyu --cxx17ns -Xiwyu --max_line_length=200 " + args
518 task_queue.put((invocation, moduleRules))
521 if len(failed_files):
524 except KeyboardInterrupt:
525 print('\nCtrl-C detected, goodbye.')
528 sys.exit(return_code)
532 parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
533 parser.add_argument('--continue', action='store_true',
534 help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
535 parser.add_argument('Files' , nargs='*',
536 help='The files to be checked')
537 parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
538 help='Recursively search a directory for source files to check')
539 parser.add_argument('--headers', action='store_true',
540 help='Check header files. If omitted, check source files. Use with --recursive.')
541 parser.add_argument('--noexclude', action='store_true',
542 help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
543 parser.add_argument('--ns', action='store_true',
544 help='Warn about unused "using namespace" statements. '
545 'Removing these may uncover more removable headers '
546 'in a subsequent normal run')
547 parser.add_argument('--finderrors', action='store_true',
548 help='Report IWYU failures when it returns with -1 error code. '
549 'Use only for debugging this script!')
550 parser.add_argument('--fwdecl', action='store_true',
551 help='Suggest removal of obsolete forward declarations')
553 args = parser.parse_args()
561 for root, dirs, files in os.walk(args.recursive[0]):
565 if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
566 list_of_files.append(os.path.join(root,file))
568 # In fwdecl mode don't check hrc files as they contain a lot of fw declarations
569 # used in defines and iwyu (0.21 at least) can not yet understand those properly
570 if (file.endswith(".hxx") or file.endswith(".h")):
571 list_of_files.append(os.path.join(root,file))
573 if (file.endswith(".cxx") or file.endswith(".c")):
574 list_of_files.append(os.path.join(root,file))
576 list_of_files = args.Files
579 with open("compile_commands.json", 'r') as compileCommandsSock:
580 compileCommands = json.load(compileCommandsSock)
581 except FileNotFoundError:
582 print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
585 # quickly sanity check whether files with exceptions in yaml still exists
586 # only check for the module of the very first filename passed
588 # Verify there are files selected for checking, with --recursive it
589 # may happen that there are in fact no C/C++ files in a module directory
590 if not list_of_files:
591 print("No files found to check!")
594 moduleName = sorted(list_of_files)[0].split("/")[0]
595 rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
597 if os.path.exists(rulePath):
598 moduleRules = yaml.full_load(open(rulePath))
599 if "excludelist" in moduleRules.keys():
600 excludelistRules = moduleRules["excludelist"]
601 for pathname in excludelistRules.keys():
602 file = pathlib.Path(pathname)
603 if not file.exists():
604 print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
606 tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns, finderrors=args.finderrors, removefwdd=args.fwdecl)
608 if __name__ == '__main__':
611 # vim:set shiftwidth=4 softtabstop=4 expandtab: