clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py

   1 #!/usr/bin/env python3
   2 #
   3 # ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ===-----------------------------------------------------------------------===#
  10
  11 r"""
  12 ClangTidy Diff Checker
  13 ======================
  14
  15 This script reads input from a unified diff, runs clang-tidy on all changed
  16 files and outputs clang-tidy warnings in changed lines only. This is useful to
  17 detect clang-tidy regressions in the lines touched by a specific patch.
  18 Example usage for git/svn users:
  19
  20   git diff -U0 HEAD^ | clang-tidy-diff.py -p1
  21   svn diff --diff-cmd=diff -x-U0 | \
  22       clang-tidy-diff.py -fix -checks=-*,modernize-use-override
  23
  24 """
  25
  26 import argparse
  27 import glob
  28 import json
  29 import multiprocessing
  30 import os
  31 import re
  32 import shutil
  33 import subprocess
  34 import sys
  35 import tempfile
  36 import threading
  37 import traceback
  38
  39 try:
  40     import yaml
  41 except ImportError:
  42     yaml = None
  43
  44 is_py2 = sys.version[0] == "2"
  45
  46 if is_py2:
  47     import Queue as queue
  48 else:
  49     import queue as queue
  50
  51
  52 def run_tidy(task_queue, lock, timeout, failed_files):
  53     watchdog = None
  54     while True:
  55         command = task_queue.get()
  56         try:
  57             proc = subprocess.Popen(
  58                 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
  59             )
  60
  61             if timeout is not None:
  62                 watchdog = threading.Timer(timeout, proc.kill)
  63                 watchdog.start()
  64
  65             stdout, stderr = proc.communicate()
  66             if proc.returncode != 0:
  67                 if proc.returncode < 0:
  68                     msg = "Terminated by signal %d : %s\n" % (
  69                         -proc.returncode,
  70                         " ".join(command),
  71                     )
  72                     stderr += msg.encode("utf-8")
  73                 failed_files.append(command)
  74
  75             with lock:
  76                 sys.stdout.write(stdout.decode("utf-8") + "\n")
  77                 sys.stdout.flush()
  78                 if stderr:
  79                     sys.stderr.write(stderr.decode("utf-8") + "\n")
  80                     sys.stderr.flush()
  81         except Exception as e:
  82             with lock:
  83                 sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
  84         finally:
  85             with lock:
  86                 if not (timeout is None or watchdog is None):
  87                     if not watchdog.is_alive():
  88                         sys.stderr.write(
  89                             "Terminated by timeout: " + " ".join(command) + "\n"
  90                         )
  91                     watchdog.cancel()
  92             task_queue.task_done()
  93
  94
  95 def start_workers(max_tasks, tidy_caller, arguments):
  96     for _ in range(max_tasks):
  97         t = threading.Thread(target=tidy_caller, args=arguments)
  98         t.daemon = True
  99         t.start()
 100
 101
 102 def merge_replacement_files(tmpdir, mergefile):
 103     """Merge all replacement files in a directory into a single file"""
 104     # The fixes suggested by clang-tidy >= 4.0.0 are given under
 105     # the top level key 'Diagnostics' in the output yaml files
 106     mergekey = "Diagnostics"
 107     merged = []
 108     for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
 109         content = yaml.safe_load(open(replacefile, "r"))
 110         if not content:
 111             continue  # Skip empty files.
 112         merged.extend(content.get(mergekey, []))
 113
 114     if merged:
 115         # MainSourceFile: The key is required by the definition inside
 116         # include/clang/Tooling/ReplacementsYaml.h, but the value
 117         # is actually never used inside clang-apply-replacements,
 118         # so we set it to '' here.
 119         output = {"MainSourceFile": "", mergekey: merged}
 120         with open(mergefile, "w") as out:
 121             yaml.safe_dump(output, out)
 122     else:
 123         # Empty the file:
 124         open(mergefile, "w").close()
 125
 126
 127 def main():
 128     parser = argparse.ArgumentParser(
 129         description="Run clang-tidy against changed files, and "
 130         "output diagnostics only for modified "
 131         "lines."
 132     )
 133     parser.add_argument(
 134         "-clang-tidy-binary",
 135         metavar="PATH",
 136         default="clang-tidy",
 137         help="path to clang-tidy binary",
 138     )
 139     parser.add_argument(
 140         "-p",
 141         metavar="NUM",
 142         default=0,
 143         help="strip the smallest prefix containing P slashes",
 144     )
 145     parser.add_argument(
 146         "-regex",
 147         metavar="PATTERN",
 148         default=None,
 149         help="custom pattern selecting file paths to check "
 150         "(case sensitive, overrides -iregex)",
 151     )
 152     parser.add_argument(
 153         "-iregex",
 154         metavar="PATTERN",
 155         default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
 156         help="custom pattern selecting file paths to check "
 157         "(case insensitive, overridden by -regex)",
 158     )
 159     parser.add_argument(
 160         "-j",
 161         type=int,
 162         default=1,
 163         help="number of tidy instances to be run in parallel.",
 164     )
 165     parser.add_argument(
 166         "-timeout", type=int, default=None, help="timeout per each file in seconds."
 167     )
 168     parser.add_argument(
 169         "-fix", action="store_true", default=False, help="apply suggested fixes"
 170     )
 171     parser.add_argument(
 172         "-checks",
 173         help="checks filter, when not specified, use clang-tidy " "default",
 174         default="",
 175     )
 176     parser.add_argument("-use-color", action="store_true", help="Use colors in output")
 177     parser.add_argument(
 178         "-path", dest="build_path", help="Path used to read a compile command database."
 179     )
 180     if yaml:
 181         parser.add_argument(
 182             "-export-fixes",
 183             metavar="FILE_OR_DIRECTORY",
 184             dest="export_fixes",
 185             help="A directory or a yaml file to store suggested fixes in, "
 186             "which can be applied with clang-apply-replacements. If the "
 187             "parameter is a directory, the fixes of each compilation unit are "
 188             "stored in individual yaml files in the directory.",
 189         )
 190     else:
 191         parser.add_argument(
 192             "-export-fixes",
 193             metavar="DIRECTORY",
 194             dest="export_fixes",
 195             help="A directory to store suggested fixes in, which can be applied "
 196             "with clang-apply-replacements. The fixes of each compilation unit are "
 197             "stored in individual yaml files in the directory.",
 198         )
 199     parser.add_argument(
 200         "-extra-arg",
 201         dest="extra_arg",
 202         action="append",
 203         default=[],
 204         help="Additional argument to append to the compiler " "command line.",
 205     )
 206     parser.add_argument(
 207         "-extra-arg-before",
 208         dest="extra_arg_before",
 209         action="append",
 210         default=[],
 211         help="Additional argument to prepend to the compiler " "command line.",
 212     )
 213     parser.add_argument(
 214         "-quiet",
 215         action="store_true",
 216         default=False,
 217         help="Run clang-tidy in quiet mode",
 218     )
 219     parser.add_argument(
 220         "-load",
 221         dest="plugins",
 222         action="append",
 223         default=[],
 224         help="Load the specified plugin in clang-tidy.",
 225     )
 226
 227     clang_tidy_args = []
 228     argv = sys.argv[1:]
 229     if "--" in argv:
 230         clang_tidy_args.extend(argv[argv.index("--") :])
 231         argv = argv[: argv.index("--")]
 232
 233     args = parser.parse_args(argv)
 234
 235     # Extract changed lines for each file.
 236     filename = None
 237     lines_by_file = {}
 238     for line in sys.stdin:
 239         match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
 240         if match:
 241             filename = match.group(2)
 242         if filename is None:
 243             continue
 244
 245         if args.regex is not None:
 246             if not re.match("^%s$" % args.regex, filename):
 247                 continue
 248         else:
 249             if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
 250                 continue
 251
 252         match = re.search("^@@.*\+(\d+)(,(\d+))?", line)
 253         if match:
 254             start_line = int(match.group(1))
 255             line_count = 1
 256             if match.group(3):
 257                 line_count = int(match.group(3))
 258             if line_count == 0:
 259                 continue
 260             end_line = start_line + line_count - 1
 261             lines_by_file.setdefault(filename, []).append([start_line, end_line])
 262
 263     if not any(lines_by_file):
 264         print("No relevant changes found.")
 265         sys.exit(0)
 266
 267     max_task_count = args.j
 268     if max_task_count == 0:
 269         max_task_count = multiprocessing.cpu_count()
 270     max_task_count = min(len(lines_by_file), max_task_count)
 271
 272     combine_fixes = False
 273     export_fixes_dir = None
 274     delete_fixes_dir = False
 275     if args.export_fixes is not None:
 276         # if a directory is given, create it if it does not exist
 277         if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
 278             args.export_fixes
 279         ):
 280             os.makedirs(args.export_fixes)
 281
 282         if not os.path.isdir(args.export_fixes):
 283             if not yaml:
 284                 raise RuntimeError(
 285                     "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
 286                 )
 287
 288             combine_fixes = True
 289
 290         if os.path.isdir(args.export_fixes):
 291             export_fixes_dir = args.export_fixes
 292
 293     if combine_fixes:
 294         export_fixes_dir = tempfile.mkdtemp()
 295         delete_fixes_dir = True
 296
 297     # Tasks for clang-tidy.
 298     task_queue = queue.Queue(max_task_count)
 299     # A lock for console output.
 300     lock = threading.Lock()
 301
 302     # List of files with a non-zero return code.
 303     failed_files = []
 304
 305     # Run a pool of clang-tidy workers.
 306     start_workers(
 307         max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files)
 308     )
 309
 310     # Form the common args list.
 311     common_clang_tidy_args = []
 312     if args.fix:
 313         common_clang_tidy_args.append("-fix")
 314     if args.checks != "":
 315         common_clang_tidy_args.append("-checks=" + args.checks)
 316     if args.quiet:
 317         common_clang_tidy_args.append("-quiet")
 318     if args.build_path is not None:
 319         common_clang_tidy_args.append("-p=%s" % args.build_path)
 320     if args.use_color:
 321         common_clang_tidy_args.append("--use-color")
 322     for arg in args.extra_arg:
 323         common_clang_tidy_args.append("-extra-arg=%s" % arg)
 324     for arg in args.extra_arg_before:
 325         common_clang_tidy_args.append("-extra-arg-before=%s" % arg)
 326     for plugin in args.plugins:
 327         common_clang_tidy_args.append("-load=%s" % plugin)
 328
 329     for name in lines_by_file:
 330         line_filter_json = json.dumps(
 331             [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")
 332         )
 333
 334         # Run clang-tidy on files containing changes.
 335         command = [args.clang_tidy_binary]
 336         command.append("-line-filter=" + line_filter_json)
 337         if args.export_fixes is not None:
 338             # Get a temporary file. We immediately close the handle so clang-tidy can
 339             # overwrite it.
 340             (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)
 341             os.close(handle)
 342             command.append("-export-fixes=" + tmp_name)
 343         command.extend(common_clang_tidy_args)
 344         command.append(name)
 345         command.extend(clang_tidy_args)
 346
 347         task_queue.put(command)
 348
 349     # Application return code
 350     return_code = 0
 351
 352     # Wait for all threads to be done.
 353     task_queue.join()
 354     # Application return code
 355     return_code = 0
 356     if failed_files:
 357         return_code = 1
 358
 359     if combine_fixes:
 360         print("Writing fixes to " + args.export_fixes + " ...")
 361         try:
 362             merge_replacement_files(export_fixes_dir, args.export_fixes)
 363         except:
 364             sys.stderr.write("Error exporting fixes.\n")
 365             traceback.print_exc()
 366             return_code = 1
 367
 368     if delete_fixes_dir:
 369         shutil.rmtree(export_fixes_dir)
 370     sys.exit(return_code)
 371
 372
 373 if __name__ == "__main__":
 374     main()