clang-tools-extra/test/clang-tidy/check_clang_tidy.py

   1 #!/usr/bin/env python3
   2 #
   3 # ===- check_clang_tidy.py - ClangTidy Test Helper ------------*- python -*--===#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ===------------------------------------------------------------------------===#
  10
  11 """
  12 ClangTidy Test Helper
  13 =====================
  14
  15 This script is used to simplify writing, running, and debugging tests compatible
  16 with llvm-lit. By default it runs clang-tidy in fix mode and uses FileCheck to
  17 verify messages and/or fixes.
  18
  19 For debugging, with --export-fixes, the tool simply exports fixes to a provided
  20 file and does not run FileCheck.
  21
  22 Extra arguments, those after the first -- if any, are passed to either
  23 clang-tidy or clang:
  24 * Arguments between the first -- and second -- are clang-tidy arguments.
  25   * May be only whitespace if there are no clang-tidy arguments.
  26   * clang-tidy's --config would go here.
  27 * Arguments after the second -- are clang arguments
  28
  29 Examples
  30 --------
  31
  32   // RUN: %check_clang_tidy %s llvm-include-order %t -- -- -isystem %S/Inputs
  33
  34 or
  35
  36   // RUN: %check_clang_tidy %s llvm-include-order --export-fixes=fixes.yaml %t -std=c++20
  37
  38 Notes
  39 -----
  40   -std=c++(98|11|14|17|20)-or-later:
  41     This flag will cause multiple runs within the same check_clang_tidy
  42     execution. Make sure you don't have shared state across these runs.
  43 """
  44
  45 import argparse
  46 import os
  47 import pathlib
  48 import re
  49 import subprocess
  50 import sys
  51
  52
  53 def write_file(file_name, text):
  54     with open(file_name, "w", encoding="utf-8") as f:
  55         f.write(text)
  56         f.truncate()
  57
  58
  59 def try_run(args, raise_error=True):
  60     try:
  61         process_output = subprocess.check_output(args, stderr=subprocess.STDOUT).decode(
  62             errors="ignore"
  63         )
  64     except subprocess.CalledProcessError as e:
  65         process_output = e.output.decode(errors="ignore")
  66         print("%s failed:\n%s" % (" ".join(args), process_output))
  67         if raise_error:
  68             raise
  69     return process_output
  70
  71
  72 # This class represents the appearance of a message prefix in a file.
  73 class MessagePrefix:
  74     def __init__(self, label):
  75         self.has_message = False
  76         self.prefixes = []
  77         self.label = label
  78
  79     def check(self, file_check_suffix, input_text):
  80         self.prefix = self.label + file_check_suffix
  81         self.has_message = self.prefix in input_text
  82         if self.has_message:
  83             self.prefixes.append(self.prefix)
  84         return self.has_message
  85
  86
  87 class CheckRunner:
  88     def __init__(self, args, extra_args):
  89         self.resource_dir = args.resource_dir
  90         self.assume_file_name = args.assume_filename
  91         self.input_file_name = args.input_file_name
  92         self.check_name = args.check_name
  93         self.temp_file_name = args.temp_file_name
  94         self.original_file_name = self.temp_file_name + ".orig"
  95         self.expect_clang_tidy_error = args.expect_clang_tidy_error
  96         self.std = args.std
  97         self.check_suffix = args.check_suffix
  98         self.input_text = ""
  99         self.has_check_fixes = False
 100         self.has_check_messages = False
 101         self.has_check_notes = False
 102         self.expect_no_diagnosis = False
 103         self.export_fixes = args.export_fixes
 104         self.fixes = MessagePrefix("CHECK-FIXES")
 105         self.messages = MessagePrefix("CHECK-MESSAGES")
 106         self.notes = MessagePrefix("CHECK-NOTES")
 107
 108         file_name_with_extension = self.assume_file_name or self.input_file_name
 109         _, extension = os.path.splitext(file_name_with_extension)
 110         if extension not in [".c", ".hpp", ".m", ".mm"]:
 111             extension = ".cpp"
 112         self.temp_file_name = self.temp_file_name + extension
 113
 114         self.clang_extra_args = []
 115         self.clang_tidy_extra_args = extra_args
 116         if "--" in extra_args:
 117             i = self.clang_tidy_extra_args.index("--")
 118             self.clang_extra_args = self.clang_tidy_extra_args[i + 1 :]
 119             self.clang_tidy_extra_args = self.clang_tidy_extra_args[:i]
 120
 121         # If the test does not specify a config style, force an empty one; otherwise
 122         # auto-detection logic can discover a ".clang-tidy" file that is not related to
 123         # the test.
 124         if not any(
 125             [re.match("^-?-config(-file)?=", arg) for arg in self.clang_tidy_extra_args]
 126         ):
 127             self.clang_tidy_extra_args.append("--config={}")
 128
 129         if extension in [".m", ".mm"]:
 130             self.clang_extra_args = [
 131                 "-fobjc-abi-version=2",
 132                 "-fobjc-arc",
 133                 "-fblocks",
 134             ] + self.clang_extra_args
 135
 136         if extension in [".cpp", ".hpp", ".mm"]:
 137             self.clang_extra_args.append("-std=" + self.std)
 138
 139         # Tests should not rely on STL being available, and instead provide mock
 140         # implementations of relevant APIs.
 141         self.clang_extra_args.append("-nostdinc++")
 142
 143         if self.resource_dir is not None:
 144             self.clang_extra_args.append("-resource-dir=%s" % self.resource_dir)
 145
 146     def read_input(self):
 147         with open(self.input_file_name, "r", encoding="utf-8") as input_file:
 148             self.input_text = input_file.read()
 149
 150     def get_prefixes(self):
 151         for suffix in self.check_suffix:
 152             if suffix and not re.match("^[A-Z0-9\\-]+$", suffix):
 153                 sys.exit(
 154                     'Only A..Z, 0..9 and "-" are allowed in check suffixes list,'
 155                     + ' but "%s" was given' % suffix
 156                 )
 157
 158             file_check_suffix = ("-" + suffix) if suffix else ""
 159
 160             has_check_fix = self.fixes.check(file_check_suffix, self.input_text)
 161             self.has_check_fixes = self.has_check_fixes or has_check_fix
 162
 163             has_check_message = self.messages.check(file_check_suffix, self.input_text)
 164             self.has_check_messages = self.has_check_messages or has_check_message
 165
 166             has_check_note = self.notes.check(file_check_suffix, self.input_text)
 167             self.has_check_notes = self.has_check_notes or has_check_note
 168
 169             if has_check_note and has_check_message:
 170                 sys.exit(
 171                     "Please use either %s or %s but not both"
 172                     % (self.notes.prefix, self.messages.prefix)
 173                 )
 174
 175             if not has_check_fix and not has_check_message and not has_check_note:
 176                 self.expect_no_diagnosis = True
 177
 178         expect_diagnosis = (
 179             self.has_check_fixes or self.has_check_messages or self.has_check_notes
 180         )
 181         if self.expect_no_diagnosis and expect_diagnosis:
 182             sys.exit(
 183                 "%s, %s or %s not found in the input"
 184                 % (
 185                     self.fixes.prefix,
 186                     self.messages.prefix,
 187                     self.notes.prefix,
 188                 )
 189             )
 190         assert expect_diagnosis or self.expect_no_diagnosis
 191
 192     def prepare_test_inputs(self):
 193         # Remove the contents of the CHECK lines to avoid CHECKs matching on
 194         # themselves.  We need to keep the comments to preserve line numbers while
 195         # avoiding empty lines which could potentially trigger formatting-related
 196         # checks.
 197         cleaned_test = re.sub("// *CHECK-[A-Z0-9\\-]*:[^\r\n]*", "//", self.input_text)
 198         write_file(self.temp_file_name, cleaned_test)
 199         write_file(self.original_file_name, cleaned_test)
 200
 201     def run_clang_tidy(self):
 202         args = (
 203             [
 204                 "clang-tidy",
 205                 self.temp_file_name,
 206             ]
 207             + [
 208                 (
 209                     "-fix"
 210                     if self.export_fixes is None
 211                     else "--export-fixes=" + self.export_fixes
 212                 )
 213             ]
 214             + [
 215                 "--checks=-*," + self.check_name,
 216             ]
 217             + self.clang_tidy_extra_args
 218             + ["--"]
 219             + self.clang_extra_args
 220         )
 221         if self.expect_clang_tidy_error:
 222             args.insert(0, "not")
 223         print("Running " + repr(args) + "...")
 224         clang_tidy_output = try_run(args)
 225         print("------------------------ clang-tidy output -----------------------")
 226         print(
 227             clang_tidy_output.encode(sys.stdout.encoding, errors="replace").decode(
 228                 sys.stdout.encoding
 229             )
 230         )
 231         print("------------------------------------------------------------------")
 232
 233         diff_output = try_run(
 234             ["diff", "-u", self.original_file_name, self.temp_file_name], False
 235         )
 236         print("------------------------------ Fixes -----------------------------")
 237         print(diff_output)
 238         print("------------------------------------------------------------------")
 239         return clang_tidy_output
 240
 241     def check_no_diagnosis(self, clang_tidy_output):
 242         if clang_tidy_output != "":
 243             sys.exit("No diagnostics were expected, but found the ones above")
 244
 245     def check_fixes(self):
 246         if self.has_check_fixes:
 247             try_run(
 248                 [
 249                     "FileCheck",
 250                     "-input-file=" + self.temp_file_name,
 251                     self.input_file_name,
 252                     "-check-prefixes=" + ",".join(self.fixes.prefixes),
 253                     "-strict-whitespace",
 254                 ]
 255             )
 256
 257     def check_messages(self, clang_tidy_output):
 258         if self.has_check_messages:
 259             messages_file = self.temp_file_name + ".msg"
 260             write_file(messages_file, clang_tidy_output)
 261             try_run(
 262                 [
 263                     "FileCheck",
 264                     "-input-file=" + messages_file,
 265                     self.input_file_name,
 266                     "-check-prefixes=" + ",".join(self.messages.prefixes),
 267                     "-implicit-check-not={{warning|error}}:",
 268                 ]
 269             )
 270
 271     def check_notes(self, clang_tidy_output):
 272         if self.has_check_notes:
 273             notes_file = self.temp_file_name + ".notes"
 274             filtered_output = [
 275                 line
 276                 for line in clang_tidy_output.splitlines()
 277                 if not ("note: FIX-IT applied" in line)
 278             ]
 279             write_file(notes_file, "\n".join(filtered_output))
 280             try_run(
 281                 [
 282                     "FileCheck",
 283                     "-input-file=" + notes_file,
 284                     self.input_file_name,
 285                     "-check-prefixes=" + ",".join(self.notes.prefixes),
 286                     "-implicit-check-not={{note|warning|error}}:",
 287                 ]
 288             )
 289
 290     def run(self):
 291         self.read_input()
 292         if self.export_fixes is None:
 293             self.get_prefixes()
 294         self.prepare_test_inputs()
 295         clang_tidy_output = self.run_clang_tidy()
 296         if self.expect_no_diagnosis:
 297             self.check_no_diagnosis(clang_tidy_output)
 298         elif self.export_fixes is None:
 299             self.check_fixes()
 300             self.check_messages(clang_tidy_output)
 301             self.check_notes(clang_tidy_output)
 302
 303
 304 CPP_STANDARDS = [
 305     "c++98",
 306     "c++11",
 307     ("c++14", "c++1y"),
 308     ("c++17", "c++1z"),
 309     ("c++20", "c++2a"),
 310     ("c++23", "c++2b"),
 311     ("c++26", "c++2c"),
 312 ]
 313 C_STANDARDS = ["c99", ("c11", "c1x"), "c17", ("c23", "c2x"), "c2y"]
 314
 315
 316 def expand_std(std):
 317     split_std, or_later, _ = std.partition("-or-later")
 318
 319     if not or_later:
 320         return [split_std]
 321
 322     for standard_list in (CPP_STANDARDS, C_STANDARDS):
 323         item = next(
 324             (
 325                 i
 326                 for i, v in enumerate(standard_list)
 327                 if (split_std in v if isinstance(v, (list, tuple)) else split_std == v)
 328             ),
 329             None,
 330         )
 331         if item is not None:
 332             return [split_std] + [
 333                 x if isinstance(x, str) else x[0] for x in standard_list[item + 1 :]
 334             ]
 335     return [std]
 336
 337
 338 def csv(string):
 339     return string.split(",")
 340
 341
 342 def parse_arguments():
 343     parser = argparse.ArgumentParser(
 344         prog=pathlib.Path(__file__).stem,
 345         description=__doc__,
 346         formatter_class=argparse.RawDescriptionHelpFormatter,
 347     )
 348     parser.add_argument("-expect-clang-tidy-error", action="store_true")
 349     parser.add_argument("-resource-dir")
 350     parser.add_argument("-assume-filename")
 351     parser.add_argument("input_file_name")
 352     parser.add_argument("check_name")
 353     parser.add_argument("temp_file_name")
 354     parser.add_argument(
 355         "-check-suffix",
 356         "-check-suffixes",
 357         default=[""],
 358         type=csv,
 359         help="comma-separated list of FileCheck suffixes",
 360     )
 361     parser.add_argument(
 362         "-export-fixes",
 363         default=None,
 364         type=str,
 365         metavar="file",
 366         help="A file to export fixes into instead of fixing.",
 367     )
 368     parser.add_argument(
 369         "-std",
 370         type=csv,
 371         default=["c++11-or-later"],
 372         help="Passed to clang. Special -or-later values are expanded.",
 373     )
 374     return parser.parse_known_args()
 375
 376
 377 def main():
 378     args, extra_args = parse_arguments()
 379
 380     abbreviated_stds = args.std
 381     for abbreviated_std in abbreviated_stds:
 382         for std in expand_std(abbreviated_std):
 383             args.std = std
 384             CheckRunner(args, extra_args).run()
 385
 386
 387 if __name__ == "__main__":
 388     main()