clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py

   1 #!/usr/bin/env python3
   2 #
   3 # ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ===-----------------------------------------------------------------------===#
  10
  11 r"""
  12 ClangTidy Diff Checker
  13 ======================
  14
  15 This script reads input from a unified diff, runs clang-tidy on all changed
  16 files and outputs clang-tidy warnings in changed lines only. This is useful to
  17 detect clang-tidy regressions in the lines touched by a specific patch.
  18 Example usage for git/svn users:
  19
  20   git diff -U0 HEAD^ | clang-tidy-diff.py -p1
  21   svn diff --diff-cmd=diff -x-U0 | \
  22       clang-tidy-diff.py -fix -checks=-*,modernize-use-override
  23
  24 """
  25
  26 import argparse
  27 import glob
  28 import json
  29 import multiprocessing
  30 import os
  31 import re
  32 import shutil
  33 import subprocess
  34 import sys
  35 import tempfile
  36 import threading
  37 import traceback
  38
  39 try:
  40     import yaml
  41 except ImportError:
  42     yaml = None
  43
  44 is_py2 = sys.version[0] == "2"
  45
  46 if is_py2:
  47     import Queue as queue
  48 else:
  49     import queue as queue
  50
  51
  52 def run_tidy(task_queue, lock, timeout, failed_files):
  53     watchdog = None
  54     while True:
  55         command = task_queue.get()
  56         try:
  57             proc = subprocess.Popen(
  58                 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
  59             )
  60
  61             if timeout is not None:
  62                 watchdog = threading.Timer(timeout, proc.kill)
  63                 watchdog.start()
  64
  65             stdout, stderr = proc.communicate()
  66             if proc.returncode != 0:
  67                 if proc.returncode < 0:
  68                     msg = "Terminated by signal %d : %s\n" % (
  69                         -proc.returncode,
  70                         " ".join(command),
  71                     )
  72                     stderr += msg.encode("utf-8")
  73                 failed_files.append(command)
  74
  75             with lock:
  76                 sys.stdout.write(stdout.decode("utf-8") + "\n")
  77                 sys.stdout.flush()
  78                 if stderr:
  79                     sys.stderr.write(stderr.decode("utf-8") + "\n")
  80                     sys.stderr.flush()
  81         except Exception as e:
  82             with lock:
  83                 sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
  84         finally:
  85             with lock:
  86                 if not (timeout is None or watchdog is None):
  87                     if not watchdog.is_alive():
  88                         sys.stderr.write(
  89                             "Terminated by timeout: " + " ".join(command) + "\n"
  90                         )
  91                     watchdog.cancel()
  92             task_queue.task_done()
  93
  94
  95 def start_workers(max_tasks, tidy_caller, arguments):
  96     for _ in range(max_tasks):
  97         t = threading.Thread(target=tidy_caller, args=arguments)
  98         t.daemon = True
  99         t.start()
 100
 101
 102 def merge_replacement_files(tmpdir, mergefile):
 103     """Merge all replacement files in a directory into a single file"""
 104     # The fixes suggested by clang-tidy >= 4.0.0 are given under
 105     # the top level key 'Diagnostics' in the output yaml files
 106     mergekey = "Diagnostics"
 107     merged = []
 108     for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
 109         content = yaml.safe_load(open(replacefile, "r"))
 110         if not content:
 111             continue  # Skip empty files.
 112         merged.extend(content.get(mergekey, []))
 113
 114     if merged:
 115         # MainSourceFile: The key is required by the definition inside
 116         # include/clang/Tooling/ReplacementsYaml.h, but the value
 117         # is actually never used inside clang-apply-replacements,
 118         # so we set it to '' here.
 119         output = {"MainSourceFile": "", mergekey: merged}
 120         with open(mergefile, "w") as out:
 121             yaml.safe_dump(output, out)
 122     else:
 123         # Empty the file:
 124         open(mergefile, "w").close()
 125
 126
 127 def main():
 128     parser = argparse.ArgumentParser(
 129         description="Run clang-tidy against changed files, and "
 130         "output diagnostics only for modified "
 131         "lines."
 132     )
 133     parser.add_argument(
 134         "-clang-tidy-binary",
 135         metavar="PATH",
 136         default="clang-tidy",
 137         help="path to clang-tidy binary",
 138     )
 139     parser.add_argument(
 140         "-p",
 141         metavar="NUM",
 142         default=0,
 143         help="strip the smallest prefix containing P slashes",
 144     )
 145     parser.add_argument(
 146         "-regex",
 147         metavar="PATTERN",
 148         default=None,
 149         help="custom pattern selecting file paths to check "
 150         "(case sensitive, overrides -iregex)",
 151     )
 152     parser.add_argument(
 153         "-iregex",
 154         metavar="PATTERN",
 155         default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
 156         help="custom pattern selecting file paths to check "
 157         "(case insensitive, overridden by -regex)",
 158     )
 159     parser.add_argument(
 160         "-j",
 161         type=int,
 162         default=1,
 163         help="number of tidy instances to be run in parallel.",
 164     )
 165     parser.add_argument(
 166         "-timeout", type=int, default=None, help="timeout per each file in seconds."
 167     )
 168     parser.add_argument(
 169         "-fix", action="store_true", default=False, help="apply suggested fixes"
 170     )
 171     parser.add_argument(
 172         "-checks",
 173         help="checks filter, when not specified, use clang-tidy " "default",
 174         default="",
 175     )
 176     parser.add_argument(
 177         "-config-file",
 178         dest="config_file",
 179         help="Specify the path of .clang-tidy or custom config file",
 180         default="",
 181     )
 182     parser.add_argument("-use-color", action="store_true", help="Use colors in output")
 183     parser.add_argument(
 184         "-path", dest="build_path", help="Path used to read a compile command database."
 185     )
 186     if yaml:
 187         parser.add_argument(
 188             "-export-fixes",
 189             metavar="FILE_OR_DIRECTORY",
 190             dest="export_fixes",
 191             help="A directory or a yaml file to store suggested fixes in, "
 192             "which can be applied with clang-apply-replacements. If the "
 193             "parameter is a directory, the fixes of each compilation unit are "
 194             "stored in individual yaml files in the directory.",
 195         )
 196     else:
 197         parser.add_argument(
 198             "-export-fixes",
 199             metavar="DIRECTORY",
 200             dest="export_fixes",
 201             help="A directory to store suggested fixes in, which can be applied "
 202             "with clang-apply-replacements. The fixes of each compilation unit are "
 203             "stored in individual yaml files in the directory.",
 204         )
 205     parser.add_argument(
 206         "-extra-arg",
 207         dest="extra_arg",
 208         action="append",
 209         default=[],
 210         help="Additional argument to append to the compiler " "command line.",
 211     )
 212     parser.add_argument(
 213         "-extra-arg-before",
 214         dest="extra_arg_before",
 215         action="append",
 216         default=[],
 217         help="Additional argument to prepend to the compiler " "command line.",
 218     )
 219     parser.add_argument(
 220         "-quiet",
 221         action="store_true",
 222         default=False,
 223         help="Run clang-tidy in quiet mode",
 224     )
 225     parser.add_argument(
 226         "-load",
 227         dest="plugins",
 228         action="append",
 229         default=[],
 230         help="Load the specified plugin in clang-tidy.",
 231     )
 232     parser.add_argument(
 233         "-allow-no-checks",
 234         action="store_true",
 235         help="Allow empty enabled checks.",
 236     )
 237
 238     clang_tidy_args = []
 239     argv = sys.argv[1:]
 240     if "--" in argv:
 241         clang_tidy_args.extend(argv[argv.index("--") :])
 242         argv = argv[: argv.index("--")]
 243
 244     args = parser.parse_args(argv)
 245
 246     # Extract changed lines for each file.
 247     filename = None
 248     lines_by_file = {}
 249     for line in sys.stdin:
 250         match = re.search('^\\+\\+\\+\\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
 251         if match:
 252             filename = match.group(2)
 253         if filename is None:
 254             continue
 255
 256         if args.regex is not None:
 257             if not re.match("^%s$" % args.regex, filename):
 258                 continue
 259         else:
 260             if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
 261                 continue
 262
 263         match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line)
 264         if match:
 265             start_line = int(match.group(1))
 266             line_count = 1
 267             if match.group(3):
 268                 line_count = int(match.group(3))
 269             if line_count == 0:
 270                 continue
 271             end_line = start_line + line_count - 1
 272             lines_by_file.setdefault(filename, []).append([start_line, end_line])
 273
 274     if not any(lines_by_file):
 275         print("No relevant changes found.")
 276         sys.exit(0)
 277
 278     max_task_count = args.j
 279     if max_task_count == 0:
 280         max_task_count = multiprocessing.cpu_count()
 281     max_task_count = min(len(lines_by_file), max_task_count)
 282
 283     combine_fixes = False
 284     export_fixes_dir = None
 285     delete_fixes_dir = False
 286     if args.export_fixes is not None:
 287         # if a directory is given, create it if it does not exist
 288         if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
 289             args.export_fixes
 290         ):
 291             os.makedirs(args.export_fixes)
 292
 293         if not os.path.isdir(args.export_fixes):
 294             if not yaml:
 295                 raise RuntimeError(
 296                     "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
 297                 )
 298
 299             combine_fixes = True
 300
 301         if os.path.isdir(args.export_fixes):
 302             export_fixes_dir = args.export_fixes
 303
 304     if combine_fixes:
 305         export_fixes_dir = tempfile.mkdtemp()
 306         delete_fixes_dir = True
 307
 308     # Tasks for clang-tidy.
 309     task_queue = queue.Queue(max_task_count)
 310     # A lock for console output.
 311     lock = threading.Lock()
 312
 313     # List of files with a non-zero return code.
 314     failed_files = []
 315
 316     # Run a pool of clang-tidy workers.
 317     start_workers(
 318         max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files)
 319     )
 320
 321     # Form the common args list.
 322     common_clang_tidy_args = []
 323     if args.fix:
 324         common_clang_tidy_args.append("-fix")
 325     if args.checks != "":
 326         common_clang_tidy_args.append("-checks=" + args.checks)
 327     if args.config_file != "":
 328         common_clang_tidy_args.append("-config-file=" + args.config_file)
 329     if args.quiet:
 330         common_clang_tidy_args.append("-quiet")
 331     if args.build_path is not None:
 332         common_clang_tidy_args.append("-p=%s" % args.build_path)
 333     if args.use_color:
 334         common_clang_tidy_args.append("--use-color")
 335     if args.allow_no_checks:
 336         common_clang_tidy_args.append("--allow-no-checks")
 337     for arg in args.extra_arg:
 338         common_clang_tidy_args.append("-extra-arg=%s" % arg)
 339     for arg in args.extra_arg_before:
 340         common_clang_tidy_args.append("-extra-arg-before=%s" % arg)
 341     for plugin in args.plugins:
 342         common_clang_tidy_args.append("-load=%s" % plugin)
 343
 344     for name in lines_by_file:
 345         line_filter_json = json.dumps(
 346             [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")
 347         )
 348
 349         # Run clang-tidy on files containing changes.
 350         command = [args.clang_tidy_binary]
 351         command.append("-line-filter=" + line_filter_json)
 352         if args.export_fixes is not None:
 353             # Get a temporary file. We immediately close the handle so clang-tidy can
 354             # overwrite it.
 355             (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)
 356             os.close(handle)
 357             command.append("-export-fixes=" + tmp_name)
 358         command.extend(common_clang_tidy_args)
 359         command.append(name)
 360         command.extend(clang_tidy_args)
 361
 362         task_queue.put(command)
 363
 364     # Application return code
 365     return_code = 0
 366
 367     # Wait for all threads to be done.
 368     task_queue.join()
 369     # Application return code
 370     return_code = 0
 371     if failed_files:
 372         return_code = 1
 373
 374     if combine_fixes:
 375         print("Writing fixes to " + args.export_fixes + " ...")
 376         try:
 377             merge_replacement_files(export_fixes_dir, args.export_fixes)
 378         except:
 379             sys.stderr.write("Error exporting fixes.\n")
 380             traceback.print_exc()
 381             return_code = 1
 382
 383     if delete_fixes_dir:
 384         shutil.rmtree(export_fixes_dir)
 385     sys.exit(return_code)
 386
 387
 388 if __name__ == "__main__":
 389     main()