clang-tools-extra/clang-tidy/tool/run-clang-tidy.py

   1 #!/usr/bin/env python3
   2 #
   3 # ===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ===-----------------------------------------------------------------------===#
  10 # FIXME: Integrate with clang-tidy-diff.py
  11
  12
  13 """
  14 Parallel clang-tidy runner
  15 ==========================
  16
  17 Runs clang-tidy over all files in a compilation database. Requires clang-tidy
  18 and clang-apply-replacements in $PATH.
  19
  20 Example invocations.
  21 - Run clang-tidy on all files in the current working directory with a default
  22   set of checks and show warnings in the cpp files and all project headers.
  23     run-clang-tidy.py $PWD
  24
  25 - Fix all header guards.
  26     run-clang-tidy.py -fix -checks=-*,llvm-header-guard
  27
  28 - Fix all header guards included from clang-tidy and header guards
  29   for clang-tidy headers.
  30     run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
  31                       -header-filter=extra/clang-tidy
  32
  33 Compilation database setup:
  34 http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
  35 """
  36
  37 import argparse
  38 import asyncio
  39 from dataclasses import dataclass
  40 import glob
  41 import json
  42 import multiprocessing
  43 import os
  44 import re
  45 import shutil
  46 import subprocess
  47 import sys
  48 import tempfile
  49 import time
  50 import traceback
  51 from types import ModuleType
  52 from typing import Any, Awaitable, Callable, List, Optional, TypeVar
  53
  54
  55 yaml: Optional[ModuleType] = None
  56 try:
  57     import yaml
  58 except ImportError:
  59     yaml = None
  60
  61
  62 def strtobool(val: str) -> bool:
  63     """Convert a string representation of truth to a bool following LLVM's CLI argument parsing."""
  64
  65     val = val.lower()
  66     if val in ["", "true", "1"]:
  67         return True
  68     elif val in ["false", "0"]:
  69         return False
  70
  71     # Return ArgumentTypeError so that argparse does not substitute its own error message
  72     raise argparse.ArgumentTypeError(
  73         f"'{val}' is invalid value for boolean argument! Try 0 or 1."
  74     )
  75
  76
  77 def find_compilation_database(path: str) -> str:
  78     """Adjusts the directory until a compilation database is found."""
  79     result = os.path.realpath("./")
  80     while not os.path.isfile(os.path.join(result, path)):
  81         parent = os.path.dirname(result)
  82         if result == parent:
  83             print("Error: could not find compilation database.")
  84             sys.exit(1)
  85         result = parent
  86     return result
  87
  88
  89 def get_tidy_invocation(
  90     f: str,
  91     clang_tidy_binary: str,
  92     checks: str,
  93     tmpdir: Optional[str],
  94     build_path: str,
  95     header_filter: Optional[str],
  96     allow_enabling_alpha_checkers: bool,
  97     extra_arg: List[str],
  98     extra_arg_before: List[str],
  99     quiet: bool,
 100     config_file_path: str,
 101     config: str,
 102     line_filter: Optional[str],
 103     use_color: bool,
 104     plugins: List[str],
 105     warnings_as_errors: Optional[str],
 106     exclude_header_filter: Optional[str],
 107     allow_no_checks: bool,
 108 ) -> List[str]:
 109     """Gets a command line for clang-tidy."""
 110     start = [clang_tidy_binary]
 111     if allow_enabling_alpha_checkers:
 112         start.append("-allow-enabling-analyzer-alpha-checkers")
 113     if exclude_header_filter is not None:
 114         start.append(f"--exclude-header-filter={exclude_header_filter}")
 115     if header_filter is not None:
 116         start.append(f"-header-filter={header_filter}")
 117     if line_filter is not None:
 118         start.append(f"-line-filter={line_filter}")
 119     if use_color is not None:
 120         if use_color:
 121             start.append("--use-color")
 122         else:
 123             start.append("--use-color=false")
 124     if checks:
 125         start.append(f"-checks={checks}")
 126     if tmpdir is not None:
 127         start.append("-export-fixes")
 128         # Get a temporary file. We immediately close the handle so clang-tidy can
 129         # overwrite it.
 130         (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
 131         os.close(handle)
 132         start.append(name)
 133     for arg in extra_arg:
 134         start.append(f"-extra-arg={arg}")
 135     for arg in extra_arg_before:
 136         start.append(f"-extra-arg-before={arg}")
 137     start.append(f"-p={build_path}")
 138     if quiet:
 139         start.append("-quiet")
 140     if config_file_path:
 141         start.append(f"--config-file={config_file_path}")
 142     elif config:
 143         start.append(f"-config={config}")
 144     for plugin in plugins:
 145         start.append(f"-load={plugin}")
 146     if warnings_as_errors:
 147         start.append(f"--warnings-as-errors={warnings_as_errors}")
 148     if allow_no_checks:
 149         start.append("--allow-no-checks")
 150     start.append(f)
 151     return start
 152
 153
 154 def merge_replacement_files(tmpdir: str, mergefile: str) -> None:
 155     """Merge all replacement files in a directory into a single file"""
 156     assert yaml
 157     # The fixes suggested by clang-tidy >= 4.0.0 are given under
 158     # the top level key 'Diagnostics' in the output yaml files
 159     mergekey = "Diagnostics"
 160     merged = []
 161     for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
 162         content = yaml.safe_load(open(replacefile, "r"))
 163         if not content:
 164             continue  # Skip empty files.
 165         merged.extend(content.get(mergekey, []))
 166
 167     if merged:
 168         # MainSourceFile: The key is required by the definition inside
 169         # include/clang/Tooling/ReplacementsYaml.h, but the value
 170         # is actually never used inside clang-apply-replacements,
 171         # so we set it to '' here.
 172         output = {"MainSourceFile": "", mergekey: merged}
 173         with open(mergefile, "w") as out:
 174             yaml.safe_dump(output, out)
 175     else:
 176         # Empty the file:
 177         open(mergefile, "w").close()
 178
 179
 180 def find_binary(arg: str, name: str, build_path: str) -> str:
 181     """Get the path for a binary or exit"""
 182     if arg:
 183         if shutil.which(arg):
 184             return arg
 185         else:
 186             raise SystemExit(
 187                 f"error: passed binary '{arg}' was not found or is not executable"
 188             )
 189
 190     built_path = os.path.join(build_path, "bin", name)
 191     binary = shutil.which(name) or shutil.which(built_path)
 192     if binary:
 193         return binary
 194     else:
 195         raise SystemExit(f"error: failed to find {name} in $PATH or at {built_path}")
 196
 197
 198 def apply_fixes(
 199     args: argparse.Namespace, clang_apply_replacements_binary: str, tmpdir: str
 200 ) -> None:
 201     """Calls clang-apply-fixes on a given directory."""
 202     invocation = [clang_apply_replacements_binary]
 203     invocation.append("-ignore-insert-conflict")
 204     if args.format:
 205         invocation.append("-format")
 206     if args.style:
 207         invocation.append(f"-style={args.style}")
 208     invocation.append(tmpdir)
 209     subprocess.call(invocation)
 210
 211
 212 # FIXME Python 3.12: This can be simplified out with run_with_semaphore[T](...).
 213 T = TypeVar("T")
 214
 215
 216 async def run_with_semaphore(
 217     semaphore: asyncio.Semaphore,
 218     f: Callable[..., Awaitable[T]],
 219     *args: Any,
 220     **kwargs: Any,
 221 ) -> T:
 222     async with semaphore:
 223         return await f(*args, **kwargs)
 224
 225
 226 @dataclass
 227 class ClangTidyResult:
 228     filename: str
 229     invocation: List[str]
 230     returncode: int
 231     stdout: str
 232     stderr: str
 233     elapsed: float
 234
 235
 236 async def run_tidy(
 237     args: argparse.Namespace,
 238     name: str,
 239     clang_tidy_binary: str,
 240     tmpdir: str,
 241     build_path: str,
 242 ) -> ClangTidyResult:
 243     """
 244     Runs clang-tidy on a single file and returns the result.
 245     """
 246     invocation = get_tidy_invocation(
 247         name,
 248         clang_tidy_binary,
 249         args.checks,
 250         tmpdir,
 251         build_path,
 252         args.header_filter,
 253         args.allow_enabling_alpha_checkers,
 254         args.extra_arg,
 255         args.extra_arg_before,
 256         args.quiet,
 257         args.config_file,
 258         args.config,
 259         args.line_filter,
 260         args.use_color,
 261         args.plugins,
 262         args.warnings_as_errors,
 263         args.exclude_header_filter,
 264         args.allow_no_checks,
 265     )
 266
 267     try:
 268         process = await asyncio.create_subprocess_exec(
 269             *invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
 270         )
 271         start = time.time()
 272         stdout, stderr = await process.communicate()
 273         end = time.time()
 274     except asyncio.CancelledError:
 275         process.terminate()
 276         await process.wait()
 277         raise
 278
 279     assert process.returncode is not None
 280     return ClangTidyResult(
 281         name,
 282         invocation,
 283         process.returncode,
 284         stdout.decode("UTF-8"),
 285         stderr.decode("UTF-8"),
 286         end - start,
 287     )
 288
 289
 290 async def main() -> None:
 291     parser = argparse.ArgumentParser(
 292         description="Runs clang-tidy over all files "
 293         "in a compilation database. Requires "
 294         "clang-tidy and clang-apply-replacements in "
 295         "$PATH or in your build directory."
 296     )
 297     parser.add_argument(
 298         "-allow-enabling-alpha-checkers",
 299         action="store_true",
 300         help="Allow alpha checkers from clang-analyzer.",
 301     )
 302     parser.add_argument(
 303         "-clang-tidy-binary", metavar="PATH", help="Path to clang-tidy binary."
 304     )
 305     parser.add_argument(
 306         "-clang-apply-replacements-binary",
 307         metavar="PATH",
 308         help="Path to clang-apply-replacements binary.",
 309     )
 310     parser.add_argument(
 311         "-checks",
 312         default=None,
 313         help="Checks filter, when not specified, use clang-tidy default.",
 314     )
 315     config_group = parser.add_mutually_exclusive_group()
 316     config_group.add_argument(
 317         "-config",
 318         default=None,
 319         help="Specifies a configuration in YAML/JSON format: "
 320         "  -config=\"{Checks: '*', "
 321         '                       CheckOptions: {x: y}}" '
 322         "When the value is empty, clang-tidy will "
 323         "attempt to find a file named .clang-tidy for "
 324         "each source file in its parent directories.",
 325     )
 326     config_group.add_argument(
 327         "-config-file",
 328         default=None,
 329         help="Specify the path of .clang-tidy or custom config "
 330         "file: e.g. -config-file=/some/path/myTidyConfigFile. "
 331         "This option internally works exactly the same way as "
 332         "-config option after reading specified config file. "
 333         "Use either -config-file or -config, not both.",
 334     )
 335     parser.add_argument(
 336         "-exclude-header-filter",
 337         default=None,
 338         help="Regular expression matching the names of the "
 339         "headers to exclude diagnostics from. Diagnostics from "
 340         "the main file of each translation unit are always "
 341         "displayed.",
 342     )
 343     parser.add_argument(
 344         "-header-filter",
 345         default=None,
 346         help="Regular expression matching the names of the "
 347         "headers to output diagnostics from. Diagnostics from "
 348         "the main file of each translation unit are always "
 349         "displayed.",
 350     )
 351     parser.add_argument(
 352         "-source-filter",
 353         default=None,
 354         help="Regular expression matching the names of the "
 355         "source files from compilation database to output "
 356         "diagnostics from.",
 357     )
 358     parser.add_argument(
 359         "-line-filter",
 360         default=None,
 361         help="List of files with line ranges to filter the warnings.",
 362     )
 363     if yaml:
 364         parser.add_argument(
 365             "-export-fixes",
 366             metavar="file_or_directory",
 367             dest="export_fixes",
 368             help="A directory or a yaml file to store suggested fixes in, "
 369             "which can be applied with clang-apply-replacements. If the "
 370             "parameter is a directory, the fixes of each compilation unit are "
 371             "stored in individual yaml files in the directory.",
 372         )
 373     else:
 374         parser.add_argument(
 375             "-export-fixes",
 376             metavar="directory",
 377             dest="export_fixes",
 378             help="A directory to store suggested fixes in, which can be applied "
 379             "with clang-apply-replacements. The fixes of each compilation unit are "
 380             "stored in individual yaml files in the directory.",
 381         )
 382     parser.add_argument(
 383         "-j",
 384         type=int,
 385         default=0,
 386         help="Number of tidy instances to be run in parallel.",
 387     )
 388     parser.add_argument(
 389         "files",
 390         nargs="*",
 391         default=[".*"],
 392         help="Files to be processed (regex on path).",
 393     )
 394     parser.add_argument("-fix", action="store_true", help="apply fix-its.")
 395     parser.add_argument(
 396         "-format", action="store_true", help="Reformat code after applying fixes."
 397     )
 398     parser.add_argument(
 399         "-style",
 400         default="file",
 401         help="The style of reformat code after applying fixes.",
 402     )
 403     parser.add_argument(
 404         "-use-color",
 405         type=strtobool,
 406         nargs="?",
 407         const=True,
 408         help="Use colors in diagnostics, overriding clang-tidy's"
 409         " default behavior. This option overrides the 'UseColor"
 410         "' option in .clang-tidy file, if any.",
 411     )
 412     parser.add_argument(
 413         "-p", dest="build_path", help="Path used to read a compile command database."
 414     )
 415     parser.add_argument(
 416         "-extra-arg",
 417         dest="extra_arg",
 418         action="append",
 419         default=[],
 420         help="Additional argument to append to the compiler command line.",
 421     )
 422     parser.add_argument(
 423         "-extra-arg-before",
 424         dest="extra_arg_before",
 425         action="append",
 426         default=[],
 427         help="Additional argument to prepend to the compiler command line.",
 428     )
 429     parser.add_argument(
 430         "-quiet", action="store_true", help="Run clang-tidy in quiet mode."
 431     )
 432     parser.add_argument(
 433         "-load",
 434         dest="plugins",
 435         action="append",
 436         default=[],
 437         help="Load the specified plugin in clang-tidy.",
 438     )
 439     parser.add_argument(
 440         "-warnings-as-errors",
 441         default=None,
 442         help="Upgrades warnings to errors. Same format as '-checks'.",
 443     )
 444     parser.add_argument(
 445         "-allow-no-checks",
 446         action="store_true",
 447         help="Allow empty enabled checks.",
 448     )
 449     args = parser.parse_args()
 450
 451     db_path = "compile_commands.json"
 452
 453     if args.build_path is not None:
 454         build_path = args.build_path
 455     else:
 456         # Find our database
 457         build_path = find_compilation_database(db_path)
 458
 459     clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", build_path)
 460
 461     if args.fix:
 462         clang_apply_replacements_binary = find_binary(
 463             args.clang_apply_replacements_binary, "clang-apply-replacements", build_path
 464         )
 465
 466     combine_fixes = False
 467     export_fixes_dir: Optional[str] = None
 468     delete_fixes_dir = False
 469     if args.export_fixes is not None:
 470         # if a directory is given, create it if it does not exist
 471         if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
 472             args.export_fixes
 473         ):
 474             os.makedirs(args.export_fixes)
 475
 476         if not os.path.isdir(args.export_fixes):
 477             if not yaml:
 478                 raise RuntimeError(
 479                     "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
 480                 )
 481
 482             combine_fixes = True
 483
 484         if os.path.isdir(args.export_fixes):
 485             export_fixes_dir = args.export_fixes
 486
 487     if export_fixes_dir is None and (args.fix or combine_fixes):
 488         export_fixes_dir = tempfile.mkdtemp()
 489         delete_fixes_dir = True
 490
 491     try:
 492         invocation = get_tidy_invocation(
 493             "",
 494             clang_tidy_binary,
 495             args.checks,
 496             None,
 497             build_path,
 498             args.header_filter,
 499             args.allow_enabling_alpha_checkers,
 500             args.extra_arg,
 501             args.extra_arg_before,
 502             args.quiet,
 503             args.config_file,
 504             args.config,
 505             args.line_filter,
 506             args.use_color,
 507             args.plugins,
 508             args.warnings_as_errors,
 509             args.exclude_header_filter,
 510             args.allow_no_checks,
 511         )
 512         invocation.append("-list-checks")
 513         invocation.append("-")
 514         # Even with -quiet we still want to check if we can call clang-tidy.
 515         subprocess.check_call(
 516             invocation, stdout=subprocess.DEVNULL if args.quiet else None
 517         )
 518     except:
 519         print("Unable to run clang-tidy.", file=sys.stderr)
 520         sys.exit(1)
 521
 522     # Load the database and extract all files.
 523     with open(os.path.join(build_path, db_path)) as f:
 524         database = json.load(f)
 525     files = {os.path.abspath(os.path.join(e["directory"], e["file"])) for e in database}
 526     number_files_in_database = len(files)
 527
 528     # Filter source files from compilation database.
 529     if args.source_filter:
 530         try:
 531             source_filter_re = re.compile(args.source_filter)
 532         except:
 533             print(
 534                 "Error: unable to compile regex from arg -source-filter:",
 535                 file=sys.stderr,
 536             )
 537             traceback.print_exc()
 538             sys.exit(1)
 539         files = {f for f in files if source_filter_re.match(f)}
 540
 541     max_task = args.j
 542     if max_task == 0:
 543         max_task = multiprocessing.cpu_count()
 544
 545     # Build up a big regexy filter from all command line arguments.
 546     file_name_re = re.compile("|".join(args.files))
 547     files = {f for f in files if file_name_re.search(f)}
 548
 549     print(
 550         "Running clang-tidy for",
 551         len(files),
 552         "files out of",
 553         number_files_in_database,
 554         "in compilation database ...",
 555     )
 556
 557     returncode = 0
 558     semaphore = asyncio.Semaphore(max_task)
 559     tasks = [
 560         asyncio.create_task(
 561             run_with_semaphore(
 562                 semaphore,
 563                 run_tidy,
 564                 args,
 565                 f,
 566                 clang_tidy_binary,
 567                 export_fixes_dir,
 568                 build_path,
 569             )
 570         )
 571         for f in files
 572     ]
 573
 574     try:
 575         for i, coro in enumerate(asyncio.as_completed(tasks)):
 576             result = await coro
 577             if result.returncode != 0:
 578                 returncode = 1
 579                 if result.returncode < 0:
 580                     result.stderr += f"{result.filename}: terminated by signal {-result.returncode}\n"
 581             progress = f"[{i + 1: >{len(f'{len(files)}')}}/{len(files)}]"
 582             runtime = f"[{result.elapsed:.1f}s]"
 583             print(f"{progress}{runtime} {' '.join(result.invocation)}")
 584             if result.stdout:
 585                 print(result.stdout, end=("" if result.stderr else "\n"))
 586             if result.stderr:
 587                 print(result.stderr)
 588     except asyncio.CancelledError:
 589         print("\nCtrl-C detected, goodbye.")
 590         for task in tasks:
 591             task.cancel()
 592         if delete_fixes_dir:
 593             assert export_fixes_dir
 594             shutil.rmtree(export_fixes_dir)
 595         return
 596
 597     if combine_fixes:
 598         print(f"Writing fixes to {args.export_fixes} ...")
 599         try:
 600             assert export_fixes_dir
 601             merge_replacement_files(export_fixes_dir, args.export_fixes)
 602         except:
 603             print("Error exporting fixes.\n", file=sys.stderr)
 604             traceback.print_exc()
 605             returncode = 1
 606
 607     if args.fix:
 608         print("Applying fixes ...")
 609         try:
 610             assert export_fixes_dir
 611             apply_fixes(args, clang_apply_replacements_binary, export_fixes_dir)
 612         except:
 613             print("Error applying fixes.\n", file=sys.stderr)
 614             traceback.print_exc()
 615             returncode = 1
 616
 617     if delete_fixes_dir:
 618         assert export_fixes_dir
 619         shutil.rmtree(export_fixes_dir)
 620     sys.exit(returncode)
 621
 622
 623 if __name__ == "__main__":
 624     try:
 625         asyncio.run(main())
 626     except KeyboardInterrupt:
 627         pass