tdf#154285 Check upper bound of arguments in SbRtl_Minute function
[LibreOffice.git] / bin / find-unneeded-includes
blobec0a620c90907ca618d4b5984e695a087b8ca263
1 #!/usr/bin/env python3
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 # This parses the output of 'include-what-you-use', focusing on just removing
8 # not needed includes and providing a relatively conservative output by
9 # filtering out a number of LibreOffice-specific false positives.
11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
12 # you can generate one with 'make vim-ide-integration'.
14 # Design goals:
15 # - excludelist mechanism, so a warning is either fixed or excluded
16 # - works in a plugins-enabled clang build
17 # - no custom configure options required
18 # - no need to generate a dummy library to build a header
20 import json
21 import multiprocessing
22 import os
23 import queue
24 import re
25 import subprocess
26 import sys
27 import threading
28 import yaml
29 import argparse
30 import pathlib
33 def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
34     # global rules
36     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
37     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
38         hdl = include.replace(".hpp", ".hdl")
39         if hdl in toAdd:
40             return True
42     # Avoid debug STL.
43     debugStl = {
44         "array": ("debug/array", ),
45         "bitset": ("debug/bitset", ),
46         "deque": ("debug/deque", ),
47         "forward_list": ("debug/forward_list", ),
48         "list": ("debug/list", ),
49         "map": ("debug/map.h", "debug/multimap.h"),
50         "set": ("debug/set.h", "debug/multiset.h"),
51         "unordered_map": ("debug/unordered_map", ),
52         "unordered_set": ("debug/unordered_set", ),
53         "vector": ("debug/vector", ),
54     }
55     for k, values in debugStl.items():
56         if include == k:
57             for value in values:
58                 if value in toAdd:
59                     return True
61     # Avoid proposing to use libstdc++ internal headers.
62     bits = {
63         "exception": "bits/exception.h",
64         "memory": "bits/shared_ptr.h",
65         "functional": "bits/std_function.h",
66         "cmath": "bits/std_abs.h",
67         "ctime": "bits/types/clock_t.h",
68         "cstdint": "bits/stdint-uintn.h",
69     }
70     for k, v in bits.items():
71         if include == k and v in toAdd:
72             return True
74     # Avoid proposing o3tl fw declaration
75     o3tl = {
76         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
77         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
78     }
79     for k, v, in o3tl.items():
80         if include == k and v in toAdd:
81             return True
83     # Follow boost documentation.
84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
85         return True
86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
87         return True
88     if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
89         return True
90     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
91         return True
92     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
93         return True
94     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
95         return True
97     # Avoid .hxx to .h proposals in basic css/uno/* API
98     unoapi = {
99         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
100         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
101         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
102         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
103     }
104     for k, v in unoapi.items():
105         if include == k and v in toAdd:
106             return True
108     # 3rd-party, non-self-contained headers.
109     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
110         return True
111     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
112         return True
113     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
114         return True
116     noRemove = (
117         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
118         # removing this.
119         "sal/config.h",
120         # Works around a build breakage specific to the broken Android
121         # toolchain.
122         "android/compatibility.hxx",
123         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
124         "osl/endian.h",
125     )
126     if include in noRemove:
127         return True
129     # Ignore when <foo> is to be replaced with "foo".
130     if include in toAdd:
131         return True
133     fileName = os.path.relpath(absFileName, os.getcwd())
135     # Skip headers used only for compile test
136     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
137         if include.endswith(".hpp"):
138             return True
140     # yaml rules, except when --noexclude is given
142     if "excludelist" in moduleRules.keys() and not noexclude:
143         excludelistRules = moduleRules["excludelist"]
144         if fileName in excludelistRules.keys():
145             if include in excludelistRules[fileName]:
146                 return True
148     return False
151 def unwrapInclude(include):
152     # Drop <> or "" around the include.
153     return include[1:-1]
156 def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces, finderrors, removefwdd):
157     inAdd = False
158     toAdd = []
159     inRemove = False
160     toRemove = []
161     inFull = False
162     currentFileName = None
164     for line in iwyuOutput:
165         line = line.strip()
167         # Bail out if IWYU gave an error due to non self-containedness
168         if re.match ("(.*): error: (.*)", line):
169             return -1
171         # Bail out if we are in finderrors mode
172         if finderrors:
173             return -2
175         if len(line) == 0:
176             if inRemove:
177                 inRemove = False
178                 continue
179             if inAdd:
180                 inAdd = False
181                 continue
182             if inFull:
183                 inFull = False
184                 continue
186         shouldAdd = fileName + " should add these lines:"
187         match = re.match(shouldAdd, line)
188         if match:
189             currentFileName = match.group(0).split(' ')[0]
190             inAdd = True
191             continue
193         shouldRemove = fileName + " should remove these lines:"
194         match = re.match(shouldRemove, line)
195         if match:
196             currentFileName = match.group(0).split(' ')[0]
197             inRemove = True
198             continue
200         if checknamespaces:
201             match = re.match("The full include-list for " + fileName, line)
202             if match:
203                 inFull = True
204                 continue
206         if inAdd:
207             match = re.match('#include ([^ ]+)', line)
208             if match:
209                 include = unwrapInclude(match.group(1))
210                 toAdd.append(include)
211             else:
212                 # Forward declaration.
213                 toAdd.append(line)
215         if inRemove and not checknamespaces:
216             if not removefwdd:
217                 match = re.match("- #include (.*)  // lines (.*)-.*", line)
218                 if match:
219                     include = unwrapInclude(match.group(1))
220                     lineno = match.group(2)
221                     if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
222                         toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
223                     continue
224             else:
225                 # Search for obsolete forward declarations, but not header -> fwdecl replacements
226                 match = re.match("- (.*;(?: })*)*  // lines (.*)-.*", line)
227                 if match:
228                     fwdDecl = match.group(1)
229                     if fwdDecl.endswith(";"):
230                         # Remove trailing semicolon.
231                         fwdDecl = fwdDecl[:-1]
232                     lineno = match.group(2)
233                     if not ignoreRemoval(fwdDecl, toAdd, currentFileName, moduleRules, noexclude):
234                         toRemove.append("%s:%s: %s" % (currentFileName, lineno, fwdDecl))
235                     continue
237         if inFull:
238             if checknamespaces:
239                 # match for all possible URE/UNO namespaces, created with:
240                 # find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr '\012' '|'
241                 # find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr '\012' '|'
242                 # and ooo::vba namespaces
243                 # plus a few popular ones about other modules
244                 ns = re.compile(
245                                 '.*for\ ('
246                                     # URE namespaces
247                                     'beans|'
248                                     'bridge|oleautomation|'
249                                     'connection|'
250                                     'container|'
251                                     'io|'
252                                     'java|'
253                                     'lang|'
254                                     'loader|'
255                                     'reflection|'
256                                     'registry|'
257                                     'script|'
258                                     'security|'
259                                     'task|'
260                                     'uno|'
261                                     'uri|'
262                                     'util|'
263                                     # UNO namespaces
264                                     'accessibility|'
265                                     'animations|'
266                                     'auth|'
267                                     'awt|tab|tree|grid|'
268                                     'chart|'
269                                     'chart2|data|'
270                                     'configuration|bootstrap|backend|xml|'
271                                     'cui|'
272                                     'datatransfer|clipboard|dnd|'
273                                     'deployment|test|ui|'
274                                     'document|'
275                                     'drawing|framework|'
276                                     'embed|'
277                                     'form|binding|runtime|control|inspection|submission|component|validation|'
278                                     'formula|'
279                                     'frame|status|'
280                                     'gallery|'
281                                     'geometry|'
282                                     'graphic|'
283                                     'i18n|'
284                                     'image|'
285                                     'inspection|'
286                                     'ldap|'
287                                     'linguistic2|'
288                                     'logging|'
289                                     'mail|'
290                                     'media|'
291                                     'mozilla|'
292                                     'office|'
293                                     'packages|zip|manifest|'
294                                     'presentation|textfield|'
295                                     'qa|'
296                                     'rdf|'
297                                     'rendering|'
298                                     'report|inspection|meta|'
299                                     'resource|'
300                                     'scanner|'
301                                     'script|vba|browse|provider|'
302                                     'sdb|application|tools|'
303                                     'sdbc|'
304                                     'sdbcx|'
305                                     'security|'
306                                     'setup|'
307                                     'sheet|opencl|'
308                                     'smarttags|'
309                                     'style|'
310                                     'svg|'
311                                     'system|windows|'
312                                     'table|'
313                                     'task|'
314                                     'text|textfield|docinfo|fieldmaster|'
315                                     'tiledrendering|'
316                                     'ucb|'
317                                     'ui|dialogs|test|'
318                                     'util|'
319                                     'view|'
320                                     'xforms|'
321                                     'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|'
322                                     'xsd|'
323                                      # ooo::vba and its namespaces
324                                     'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|'
325                                      # use of module namespaces, as spotted in the code
326                                     'analysis|pricing' # sca internals
327                                     'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals
328                                     'basegfx|utils|'
329                                     'boost|posix_time|gregorian'
330                                     'cairo|'
331                                     'canvas|'
332                                     'chelp|'
333                                     'comphelper|'
334                                     'connectivity|'
335                                     'cpp|java|' # for codemaker::
336                                     'cppu|'
337                                     'dbaccess|dbahsql|dbaui|dbtools|'
338                                     'desktop|dp_misc|'
339                                     'drawinglayer|attribute|geometry|primitive2d|processor2d|'
340                                     'editeng|'
341                                     'emscripten|'
342                                     'formula|'
343                                     'framework|'
344                                     'frm|'
345                                     'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal
346                                     'i18npool|'
347                                     'internal|ColorComponentTag|' # for slideshow internals
348                                     'jfw_plugin|'
349                                     'jni_uno|'
350                                     'librevenge|'
351                                     'linguistic|'
352                                     'lok|'
353                                     'mtv|' # for mdds::mtv
354                                     'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal
355                                     'o3tl|'
356                                     'odfflatxml|' # filter internal
357                                     'oox|core|drawingml|ole|vml|'
358                                     'OpenStormBento|'
359                                     'osl|'
360                                     'pdfi|pdfparse|'
361                                     'ppt|'
362                                     'pyuno|'
363                                     'reportdesign|'
364                                     'rptui|'
365                                     'rtl|math|textenc|'
366                                     'salhelper|'
367                                     'sax_fastparser|'
368                                     'sax|' # for xml::sax
369                                     'sc|'
370                                     'SchXMLTools|' # for xmloff
371                                     'sd|slidesorter|cache|controller|model|view|'
372                                     'sf_misc|'
373                                     'sfx2|DocTempl|'
374                                     'sidebar|' # for sfx2::sidebar
375                                     'skeletonmaker|'
376                                     'star|' # for com::sun::star
377                                     'std|chrono_literals|literals|'
378                                     'stoc_sec|'
379                                     'store|'
380                                     'svl|impl|'
381                                     'svt|'
382                                     'svtools|'
383                                     'svx|sdr|contact|table|'
384                                     'sw|access|annotation|mark|types|util|'
385                                     'toolkit|'
386                                     'treeview|'
387                                     'ucbhelper|'
388                                     'unodevtools'
389                                     'unopkg|'
390                                     'util|db|qe|' # for xmlsearch::
391                                     'utl|'
392                                     'vcl|psp|x11|'
393                                     'writerfilter|'
394                                     'xforms|'
395                                     'xmloff|token|EnhancedCustomShapeToken' # for xmloff::
396                                     'ZipUtils'
397                                     ')$', re.VERBOSE
398                                 )
400                 reason = re.match(ns, line)
401                 if reason:
402                     # Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used
403                     # otherwise the used classes name would show up after the '// for'
404                     # Cleaning out the respective header (if there is any
405                     # - which is not always the case) is for the next run!
406                     nameSpace = reason.group(1).split(' ')[0]
407                     print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace)
409                     # Get the row number, normal IWYU output does not contain this info
410                     subprocess.run(["git", "grep", "-n", "namespace.*[^a-zA-Z]"+nameSpace+" *;", fileName])
412     if removefwdd:
413         for remove in sorted(toRemove, key=lambda x: int(x.split(":")[1])):
414             print("ERROR: %s: remove not needed forward declaration" % remove)
415     else:
416         for remove in sorted(toRemove, key=lambda x: int(x.split(":")[1])):
417             print("ERROR: %s: remove not needed include" % remove)
418     return len(toRemove)
421 def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces, finderrors, removefwdd):
422     while True:
423         invocation, moduleRules = task_queue.get()
424         if not len(failed_files):
425             print("[IWYU] " + invocation.split(' ')[-1])
426             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
427             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces, finderrors, removefwdd)
428             if finderrors:
429                 if p.returncode == 1:
430                     print("Running the IWYU process returned error code:\n" + invocation)
431             if retcode == -1 and not checknamespaces and not removefwdd:
432                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
433             elif retcode > 0:
434                 if not removefwdd:
435                     print("ERROR: The following command found unused includes:\n" + invocation)
436                 else:
437                     print("ERROR: The following command found unused forward declarations:\n" + invocation)
438                 if not dontstop:
439                     failed_files.append(invocation)
440         task_queue.task_done()
441     if checknamespaces:
442         # Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished
443         os.system('stty sane')
446 def isInUnoIncludeFile(path):
447     return path.startswith("include/com/") \
448             or path.startswith("include/cppu/") \
449             or path.startswith("include/cppuhelper/") \
450             or path.startswith("include/osl/") \
451             or path.startswith("include/rtl/") \
452             or path.startswith("include/sal/") \
453             or path.startswith("include/salhelper/") \
454             or path.startswith("include/systools/") \
455             or path.startswith("include/typelib/") \
456             or path.startswith("include/uno/")
459 def tidy(compileCommands, paths, dontstop, noexclude, checknamespaces, finderrors, removefwdd):
460     return_code = 0
462     try:
463         max_task = multiprocessing.cpu_count()
464         task_queue = queue.Queue(max_task)
465         failed_files = []
466         for _ in range(max_task):
467             t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude, checknamespaces, finderrors, removefwdd))
468             t.daemon = True
469             t.start()
471         for path in sorted(paths):
472             if isInUnoIncludeFile(path):
473                 continue
475             # IWYU fails on these with #error: don't use this in new code
476             if path.startswith("include/vcl/toolkit"):
477                 continue
479             moduleName = path.split("/")[0]
481             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
482             moduleRules = {}
483             if os.path.exists(rulePath):
484                 moduleRules = yaml.full_load(open(rulePath))
485             assume = None
486             pathAbs = os.path.abspath(path)
487             compileFile = pathAbs
488             matches = [i for i in compileCommands if i["file"] == compileFile]
489             if not len(matches):
490                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
491                 # code on Linux.
492                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
493                     assume = moduleRules["assumeFilename"]
494                 if assume:
495                     assumeAbs = os.path.abspath(assume)
496                     compileFile = assumeAbs
497                     matches = [i for i in compileCommands if i["file"] == compileFile]
498                     if not len(matches):
499                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
500                         continue
501                 else:
502                     print("WARNING: no compile commands for '" + path + "'")
503                     continue
505             _, _, args = matches[0]["command"].partition(" ")
506             if assume:
507                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
509             if not removefwdd:
510                 invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
511             # In --fwdecl mode we ask for fw declaration removal suggestions.
512             # In this mode obsolete fw declarations are suggested for removal.
513             # Later we ignore the header removal suggestions, which may be
514             # there because of possibility of replacement with fw declarations
515             # but those and our CI are not reliable enough yet for use
516             else:
517                 invocation = "include-what-you-use -Xiwyu --cxx17ns -Xiwyu --max_line_length=200 " + args
518             task_queue.put((invocation, moduleRules))
520         task_queue.join()
521         if len(failed_files):
522             return_code = 1
524     except KeyboardInterrupt:
525         print('\nCtrl-C detected, goodbye.')
526         os.kill(0, 9)
528     sys.exit(return_code)
531 def main(argv):
532     parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
533     parser.add_argument('--continue', action='store_true',
534                     help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
535     parser.add_argument('Files' , nargs='*',
536                     help='The files to be checked')
537     parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
538                     help='Recursively search a directory for source files to check')
539     parser.add_argument('--headers', action='store_true',
540                     help='Check header files. If omitted, check source files. Use with --recursive.')
541     parser.add_argument('--noexclude', action='store_true',
542                     help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
543     parser.add_argument('--ns', action='store_true',
544                     help='Warn about unused "using namespace" statements. '
545                          'Removing these may uncover more removable headers '
546                          'in a subsequent normal run')
547     parser.add_argument('--finderrors', action='store_true',
548                    help='Report IWYU failures when it returns with -1 error code. '
549                         'Use only for debugging this script!')
550     parser.add_argument('--fwdecl', action='store_true',
551                     help='Suggest removal of obsolete forward declarations')
553     args = parser.parse_args()
555     if not len(argv):
556         parser.print_help()
557         return
559     list_of_files = []
560     if args.recursive:
561         for root, dirs, files in os.walk(args.recursive[0]):
562             for file in files:
563                 if args.headers:
564                     if not args.fwdecl:
565                         if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
566                             list_of_files.append(os.path.join(root,file))
567                     else:
568                         # In fwdecl mode don't check hrc files as they contain a lot of fw declarations
569                         # used in defines and iwyu (0.21 at least) can not yet understand those properly
570                         if (file.endswith(".hxx") or file.endswith(".h")):
571                             list_of_files.append(os.path.join(root,file))
572                 else:
573                     if (file.endswith(".cxx") or file.endswith(".c")):
574                         list_of_files.append(os.path.join(root,file))
575     else:
576         list_of_files = args.Files
578     try:
579         with open("compile_commands.json", 'r') as compileCommandsSock:
580             compileCommands = json.load(compileCommandsSock)
581     except FileNotFoundError:
582         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
583         sys.exit(-1)
585     # quickly sanity check whether files with exceptions in yaml still exists
586     # only check for the module of the very first filename passed
588     # Verify there are files selected for checking, with --recursive it
589     # may happen that there are in fact no C/C++ files in a module directory
590     if not list_of_files:
591         print("No files found to check!")
592         sys.exit(-2)
594     moduleName = sorted(list_of_files)[0].split("/")[0]
595     rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
596     moduleRules = {}
597     if os.path.exists(rulePath):
598         moduleRules = yaml.full_load(open(rulePath))
599     if "excludelist" in moduleRules.keys():
600         excludelistRules = moduleRules["excludelist"]
601         for pathname in excludelistRules.keys():
602             file = pathlib.Path(pathname)
603             if not file.exists():
604                 print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
606     tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns, finderrors=args.finderrors, removefwdd=args.fwdecl)
608 if __name__ == '__main__':
609     main(sys.argv[1:])
611 # vim:set shiftwidth=4 softtabstop=4 expandtab: