clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp

   1 //===-- ClangIncludeFixer.cpp - Standalone include fixer ------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "FuzzySymbolIndex.h"
  10 #include "InMemorySymbolIndex.h"
  11 #include "IncludeFixer.h"
  12 #include "IncludeFixerContext.h"
  13 #include "SymbolIndexManager.h"
  14 #include "YamlSymbolIndex.h"
  15 #include "clang/Format/Format.h"
  16 #include "clang/Frontend/TextDiagnosticPrinter.h"
  17 #include "clang/Rewrite/Core/Rewriter.h"
  18 #include "clang/Tooling/CommonOptionsParser.h"
  19 #include "clang/Tooling/Core/Replacement.h"
  20 #include "clang/Tooling/Tooling.h"
  21 #include "llvm/Support/CommandLine.h"
  22 #include "llvm/Support/Path.h"
  23 #include "llvm/Support/YAMLTraits.h"
  24
  25 using namespace clang;
  26 using namespace llvm;
  27 using clang::include_fixer::IncludeFixerContext;
  28
  29 LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(IncludeFixerContext)
  30 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(IncludeFixerContext::HeaderInfo)
  31 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(IncludeFixerContext::QuerySymbolInfo)
  32
  33 namespace llvm {
  34 namespace yaml {
  35
  36 template <> struct MappingTraits<tooling::Range> {
  37   struct NormalizedRange {
  38     NormalizedRange(const IO &) : Offset(0), Length(0) {}
  39
  40     NormalizedRange(const IO &, const tooling::Range &R)
  41         : Offset(R.getOffset()), Length(R.getLength()) {}
  42
  43     tooling::Range denormalize(const IO &) {
  44       return tooling::Range(Offset, Length);
  45     }
  46
  47     unsigned Offset;
  48     unsigned Length;
  49   };
  50   static void mapping(IO &IO, tooling::Range &Info) {
  51     MappingNormalization<NormalizedRange, tooling::Range> Keys(IO, Info);
  52     IO.mapRequired("Offset", Keys->Offset);
  53     IO.mapRequired("Length", Keys->Length);
  54   }
  55 };
  56
  57 template <> struct MappingTraits<IncludeFixerContext::HeaderInfo> {
  58   static void mapping(IO &io, IncludeFixerContext::HeaderInfo &Info) {
  59     io.mapRequired("Header", Info.Header);
  60     io.mapRequired("QualifiedName", Info.QualifiedName);
  61   }
  62 };
  63
  64 template <> struct MappingTraits<IncludeFixerContext::QuerySymbolInfo> {
  65   static void mapping(IO &io, IncludeFixerContext::QuerySymbolInfo &Info) {
  66     io.mapRequired("RawIdentifier", Info.RawIdentifier);
  67     io.mapRequired("Range", Info.Range);
  68   }
  69 };
  70
  71 template <> struct MappingTraits<IncludeFixerContext> {
  72   static void mapping(IO &IO, IncludeFixerContext &Context) {
  73     IO.mapRequired("QuerySymbolInfos", Context.QuerySymbolInfos);
  74     IO.mapRequired("HeaderInfos", Context.HeaderInfos);
  75     IO.mapRequired("FilePath", Context.FilePath);
  76   }
  77 };
  78 } // namespace yaml
  79 } // namespace llvm
  80
  81 namespace {
  82 cl::OptionCategory IncludeFixerCategory("Tool options");
  83
  84 enum DatabaseFormatTy {
  85   fixed,     ///< Hard-coded mapping.
  86   yaml,      ///< Yaml database created by find-all-symbols.
  87   fuzzyYaml, ///< Yaml database with fuzzy-matched identifiers.
  88 };
  89
  90 cl::opt<DatabaseFormatTy> DatabaseFormat(
  91     "db", cl::desc("Specify input format"),
  92     cl::values(clEnumVal(fixed, "Hard-coded mapping"),
  93                clEnumVal(yaml, "Yaml database created by find-all-symbols"),
  94                clEnumVal(fuzzyYaml, "Yaml database, with fuzzy-matched names")),
  95     cl::init(yaml), cl::cat(IncludeFixerCategory));
  96
  97 cl::opt<std::string> Input("input",
  98                            cl::desc("String to initialize the database"),
  99                            cl::cat(IncludeFixerCategory));
 100
 101 cl::opt<std::string>
 102     QuerySymbol("query-symbol",
 103                  cl::desc("Query a given symbol (e.g. \"a::b::foo\") in\n"
 104                           "database directly without parsing the file."),
 105                  cl::cat(IncludeFixerCategory));
 106
 107 cl::opt<bool>
 108     MinimizeIncludePaths("minimize-paths",
 109                          cl::desc("Whether to minimize added include paths"),
 110                          cl::init(true), cl::cat(IncludeFixerCategory));
 111
 112 cl::opt<bool> Quiet("q", cl::desc("Reduce terminal output"), cl::init(false),
 113                     cl::cat(IncludeFixerCategory));
 114
 115 cl::opt<bool>
 116     STDINMode("stdin",
 117               cl::desc("Override source file's content (in the overlaying\n"
 118                        "virtual file system) with input from <stdin> and run\n"
 119                        "the tool on the new content with the compilation\n"
 120                        "options of the source file. This mode is currently\n"
 121                        "used for editor integration."),
 122               cl::init(false), cl::cat(IncludeFixerCategory));
 123
 124 cl::opt<bool> OutputHeaders(
 125     "output-headers",
 126     cl::desc("Print the symbol being queried and all its relevant headers in\n"
 127              "JSON format to stdout:\n"
 128              "  {\n"
 129              "    \"FilePath\": \"/path/to/foo.cc\",\n"
 130              "    \"QuerySymbolInfos\": [\n"
 131              "       {\"RawIdentifier\": \"foo\",\n"
 132              "        \"Range\": {\"Offset\": 0, \"Length\": 3}}\n"
 133              "    ],\n"
 134              "    \"HeaderInfos\": [ {\"Header\": \"\\\"foo_a.h\\\"\",\n"
 135              "                      \"QualifiedName\": \"a::foo\"} ]\n"
 136              "  }"),
 137     cl::init(false), cl::cat(IncludeFixerCategory));
 138
 139 cl::opt<std::string> InsertHeader(
 140     "insert-header",
 141     cl::desc("Insert a specific header. This should run with STDIN mode.\n"
 142              "The result is written to stdout. It is currently used for\n"
 143              "editor integration. Support YAML/JSON format:\n"
 144              "  -insert-header=\"{\n"
 145              "     FilePath: \"/path/to/foo.cc\",\n"
 146              "     QuerySymbolInfos: [\n"
 147              "       {RawIdentifier: foo,\n"
 148              "        Range: {Offset: 0, Length: 3}}\n"
 149              "     ],\n"
 150              "     HeaderInfos: [ {Headers: \"\\\"foo_a.h\\\"\",\n"
 151              "                     QualifiedName: \"a::foo\"} ]}\""),
 152     cl::init(""), cl::cat(IncludeFixerCategory));
 153
 154 cl::opt<std::string>
 155     Style("style",
 156           cl::desc("Fallback style for reformatting after inserting new\n"
 157                    "headers if there is no clang-format config file found."),
 158           cl::init("llvm"), cl::cat(IncludeFixerCategory));
 159
 160 std::unique_ptr<include_fixer::SymbolIndexManager>
 161 createSymbolIndexManager(StringRef FilePath) {
 162   using find_all_symbols::SymbolInfo;
 163
 164   auto SymbolIndexMgr = std::make_unique<include_fixer::SymbolIndexManager>();
 165   switch (DatabaseFormat) {
 166   case fixed: {
 167     // Parse input and fill the database with it.
 168     // <symbol>=<header><, header...>
 169     // Multiple symbols can be given, separated by semicolons.
 170     std::map<std::string, std::vector<std::string>> SymbolsMap;
 171     SmallVector<StringRef, 4> SemicolonSplits;
 172     StringRef(Input).split(SemicolonSplits, ";");
 173     std::vector<find_all_symbols::SymbolAndSignals> Symbols;
 174     for (StringRef Pair : SemicolonSplits) {
 175       auto Split = Pair.split('=');
 176       std::vector<std::string> Headers;
 177       SmallVector<StringRef, 4> CommaSplits;
 178       Split.second.split(CommaSplits, ",");
 179       for (size_t I = 0, E = CommaSplits.size(); I != E; ++I)
 180         Symbols.push_back(
 181             {SymbolInfo(Split.first.trim(), SymbolInfo::SymbolKind::Unknown,
 182                         CommaSplits[I].trim(), {}),
 183              // Use fake "seen" signal for tests, so first header wins.
 184              SymbolInfo::Signals(/*Seen=*/static_cast<unsigned>(E - I),
 185                                  /*Used=*/0)});
 186     }
 187     SymbolIndexMgr->addSymbolIndex([=]() {
 188       return std::make_unique<include_fixer::InMemorySymbolIndex>(Symbols);
 189     });
 190     break;
 191   }
 192   case yaml: {
 193     auto CreateYamlIdx = [=]() -> std::unique_ptr<include_fixer::SymbolIndex> {
 194       llvm::ErrorOr<std::unique_ptr<include_fixer::YamlSymbolIndex>> DB(
 195           nullptr);
 196       if (!Input.empty()) {
 197         DB = include_fixer::YamlSymbolIndex::createFromFile(Input);
 198       } else {
 199         // If we don't have any input file, look in the directory of the
 200         // first
 201         // file and its parents.
 202         SmallString<128> AbsolutePath(tooling::getAbsolutePath(FilePath));
 203         StringRef Directory = llvm::sys::path::parent_path(AbsolutePath);
 204         DB = include_fixer::YamlSymbolIndex::createFromDirectory(
 205             Directory, "find_all_symbols_db.yaml");
 206       }
 207
 208       if (!DB) {
 209         llvm::errs() << "Couldn't find YAML db: " << DB.getError().message()
 210                      << '\n';
 211         return nullptr;
 212       }
 213       return std::move(*DB);
 214     };
 215
 216     SymbolIndexMgr->addSymbolIndex(std::move(CreateYamlIdx));
 217     break;
 218   }
 219   case fuzzyYaml: {
 220     // This mode is not very useful, because we don't correct the identifier.
 221     // It's main purpose is to expose FuzzySymbolIndex to tests.
 222     SymbolIndexMgr->addSymbolIndex(
 223         []() -> std::unique_ptr<include_fixer::SymbolIndex> {
 224           auto DB = include_fixer::FuzzySymbolIndex::createFromYAML(Input);
 225           if (!DB) {
 226             llvm::errs() << "Couldn't load fuzzy YAML db: "
 227                          << llvm::toString(DB.takeError()) << '\n';
 228             return nullptr;
 229           }
 230           return std::move(*DB);
 231         });
 232     break;
 233   }
 234   }
 235   return SymbolIndexMgr;
 236 }
 237
 238 void writeToJson(llvm::raw_ostream &OS, const IncludeFixerContext& Context) {
 239   OS << "{\n"
 240      << "  \"FilePath\": \""
 241      << llvm::yaml::escape(Context.getFilePath()) << "\",\n"
 242      << "  \"QuerySymbolInfos\": [\n";
 243   for (const auto &Info : Context.getQuerySymbolInfos()) {
 244     OS << "     {\"RawIdentifier\": \"" << Info.RawIdentifier << "\",\n";
 245     OS << "      \"Range\":{";
 246     OS << "\"Offset\":" << Info.Range.getOffset() << ",";
 247     OS << "\"Length\":" << Info.Range.getLength() << "}}";
 248     if (&Info != &Context.getQuerySymbolInfos().back())
 249       OS << ",\n";
 250   }
 251   OS << "\n  ],\n";
 252   OS << "  \"HeaderInfos\": [\n";
 253   const auto &HeaderInfos = Context.getHeaderInfos();
 254   for (const auto &Info : HeaderInfos) {
 255     OS << "     {\"Header\": \"" << llvm::yaml::escape(Info.Header) << "\",\n"
 256        << "      \"QualifiedName\": \"" << Info.QualifiedName << "\"}";
 257     if (&Info != &HeaderInfos.back())
 258       OS << ",\n";
 259   }
 260   OS << "\n";
 261   OS << "  ]\n";
 262   OS << "}\n";
 263 }
 264
 265 int includeFixerMain(int argc, const char **argv) {
 266   auto ExpectedParser =
 267       tooling::CommonOptionsParser::create(argc, argv, IncludeFixerCategory);
 268   if (!ExpectedParser) {
 269     llvm::errs() << ExpectedParser.takeError();
 270     return 1;
 271   }
 272   tooling::CommonOptionsParser &options = ExpectedParser.get();
 273   tooling::ClangTool tool(options.getCompilations(),
 274                           options.getSourcePathList());
 275
 276   llvm::StringRef SourceFilePath = options.getSourcePathList().front();
 277   // In STDINMode, we override the file content with the <stdin> input.
 278   // Since `tool.mapVirtualFile` takes `StringRef`, we define `Code` outside of
 279   // the if-block so that `Code` is not released after the if-block.
 280   std::unique_ptr<llvm::MemoryBuffer> Code;
 281   if (STDINMode) {
 282     assert(options.getSourcePathList().size() == 1 &&
 283            "Expect exactly one file path in STDINMode.");
 284     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> CodeOrErr =
 285         MemoryBuffer::getSTDIN();
 286     if (std::error_code EC = CodeOrErr.getError()) {
 287       errs() << EC.message() << "\n";
 288       return 1;
 289     }
 290     Code = std::move(CodeOrErr.get());
 291     if (Code->getBufferSize() == 0)
 292       return 0;  // Skip empty files.
 293
 294     tool.mapVirtualFile(SourceFilePath, Code->getBuffer());
 295   }
 296
 297   if (!InsertHeader.empty()) {
 298     if (!STDINMode) {
 299       errs() << "Should be running in STDIN mode\n";
 300       return 1;
 301     }
 302
 303     llvm::yaml::Input yin(InsertHeader);
 304     IncludeFixerContext Context;
 305     yin >> Context;
 306
 307     const auto &HeaderInfos = Context.getHeaderInfos();
 308     assert(!HeaderInfos.empty());
 309     // We only accept one unique header.
 310     // Check all elements in HeaderInfos have the same header.
 311     bool IsUniqueHeader = std::equal(
 312         HeaderInfos.begin()+1, HeaderInfos.end(), HeaderInfos.begin(),
 313         [](const IncludeFixerContext::HeaderInfo &LHS,
 314            const IncludeFixerContext::HeaderInfo &RHS) {
 315           return LHS.Header == RHS.Header;
 316         });
 317     if (!IsUniqueHeader) {
 318       errs() << "Expect exactly one unique header.\n";
 319       return 1;
 320     }
 321
 322     // If a header has multiple symbols, we won't add the missing namespace
 323     // qualifiers because we don't know which one is exactly used.
 324     //
 325     // Check whether all elements in HeaderInfos have the same qualified name.
 326     bool IsUniqueQualifiedName = std::equal(
 327         HeaderInfos.begin() + 1, HeaderInfos.end(), HeaderInfos.begin(),
 328         [](const IncludeFixerContext::HeaderInfo &LHS,
 329            const IncludeFixerContext::HeaderInfo &RHS) {
 330           return LHS.QualifiedName == RHS.QualifiedName;
 331         });
 332     auto InsertStyle = format::getStyle(format::DefaultFormatStyle,
 333                                         Context.getFilePath(), Style);
 334     if (!InsertStyle) {
 335       llvm::errs() << llvm::toString(InsertStyle.takeError()) << "\n";
 336       return 1;
 337     }
 338     auto Replacements = clang::include_fixer::createIncludeFixerReplacements(
 339         Code->getBuffer(), Context, *InsertStyle,
 340         /*AddQualifiers=*/IsUniqueQualifiedName);
 341     if (!Replacements) {
 342       errs() << "Failed to create replacements: "
 343              << llvm::toString(Replacements.takeError()) << "\n";
 344       return 1;
 345     }
 346
 347     auto ChangedCode =
 348         tooling::applyAllReplacements(Code->getBuffer(), *Replacements);
 349     if (!ChangedCode) {
 350       llvm::errs() << llvm::toString(ChangedCode.takeError()) << "\n";
 351       return 1;
 352     }
 353     llvm::outs() << *ChangedCode;
 354     return 0;
 355   }
 356
 357   // Set up data source.
 358   std::unique_ptr<include_fixer::SymbolIndexManager> SymbolIndexMgr =
 359       createSymbolIndexManager(SourceFilePath);
 360   if (!SymbolIndexMgr)
 361     return 1;
 362
 363   // Query symbol mode.
 364   if (!QuerySymbol.empty()) {
 365     auto MatchedSymbols = SymbolIndexMgr->search(
 366         QuerySymbol, /*IsNestedSearch=*/true, SourceFilePath);
 367     for (auto &Symbol : MatchedSymbols) {
 368       std::string HeaderPath = Symbol.getFilePath().str();
 369       Symbol.SetFilePath(((HeaderPath[0] == '"' || HeaderPath[0] == '<')
 370                               ? HeaderPath
 371                               : "\"" + HeaderPath + "\""));
 372     }
 373
 374     // We leave an empty symbol range as we don't know the range of the symbol
 375     // being queried in this mode. clang-include-fixer won't add namespace
 376     // qualifiers if the symbol range is empty, which also fits this case.
 377     IncludeFixerContext::QuerySymbolInfo Symbol;
 378     Symbol.RawIdentifier = QuerySymbol;
 379     auto Context =
 380         IncludeFixerContext(SourceFilePath, {Symbol}, MatchedSymbols);
 381     writeToJson(llvm::outs(), Context);
 382     return 0;
 383   }
 384
 385   // Now run our tool.
 386   std::vector<include_fixer::IncludeFixerContext> Contexts;
 387   include_fixer::IncludeFixerActionFactory Factory(*SymbolIndexMgr, Contexts,
 388                                                    Style, MinimizeIncludePaths);
 389
 390   if (tool.run(&Factory) != 0) {
 391     // We suppress all Clang diagnostics (because they would be wrong,
 392     // clang-include-fixer does custom recovery) but still want to give some
 393     // feedback in case there was a compiler error we couldn't recover from.
 394     // The most common case for this is a #include in the file that couldn't be
 395     // found.
 396     llvm::errs() << "Fatal compiler error occurred while parsing file!"
 397                     " (incorrect include paths?)\n";
 398     return 1;
 399   }
 400
 401   assert(!Contexts.empty());
 402
 403   if (OutputHeaders) {
 404     // FIXME: Print contexts of all processing files instead of the first one.
 405     writeToJson(llvm::outs(), Contexts.front());
 406     return 0;
 407   }
 408
 409   std::vector<tooling::Replacements> FixerReplacements;
 410   for (const auto &Context : Contexts) {
 411     StringRef FilePath = Context.getFilePath();
 412     auto InsertStyle =
 413         format::getStyle(format::DefaultFormatStyle, FilePath, Style);
 414     if (!InsertStyle) {
 415       llvm::errs() << llvm::toString(InsertStyle.takeError()) << "\n";
 416       return 1;
 417     }
 418     auto Buffer = llvm::MemoryBuffer::getFile(FilePath);
 419     if (!Buffer) {
 420       errs() << "Couldn't open file: " + FilePath.str() + ": "
 421              << Buffer.getError().message() + "\n";
 422       return 1;
 423     }
 424
 425     auto Replacements = clang::include_fixer::createIncludeFixerReplacements(
 426         Buffer.get()->getBuffer(), Context, *InsertStyle);
 427     if (!Replacements) {
 428       errs() << "Failed to create replacement: "
 429              << llvm::toString(Replacements.takeError()) << "\n";
 430       return 1;
 431     }
 432     FixerReplacements.push_back(*Replacements);
 433   }
 434
 435   if (!Quiet) {
 436     for (const auto &Context : Contexts) {
 437       if (!Context.getHeaderInfos().empty()) {
 438         llvm::errs() << "Added #include "
 439                      << Context.getHeaderInfos().front().Header << " for "
 440                      << Context.getFilePath() << "\n";
 441       }
 442     }
 443   }
 444
 445   if (STDINMode) {
 446     assert(FixerReplacements.size() == 1);
 447     auto ChangedCode = tooling::applyAllReplacements(Code->getBuffer(),
 448                                                      FixerReplacements.front());
 449     if (!ChangedCode) {
 450       llvm::errs() << llvm::toString(ChangedCode.takeError()) << "\n";
 451       return 1;
 452     }
 453     llvm::outs() << *ChangedCode;
 454     return 0;
 455   }
 456
 457   // Set up a new source manager for applying the resulting replacements.
 458   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts(new DiagnosticOptions);
 459   DiagnosticsEngine Diagnostics(new DiagnosticIDs, &*DiagOpts);
 460   TextDiagnosticPrinter DiagnosticPrinter(outs(), &*DiagOpts);
 461   SourceManager SM(Diagnostics, tool.getFiles());
 462   Diagnostics.setClient(&DiagnosticPrinter, false);
 463
 464   // Write replacements to disk.
 465   Rewriter Rewrites(SM, LangOptions());
 466   for (const auto &Replacement : FixerReplacements) {
 467     if (!tooling::applyAllReplacements(Replacement, Rewrites)) {
 468       llvm::errs() << "Failed to apply replacements.\n";
 469       return 1;
 470     }
 471   }
 472   return Rewrites.overwriteChangedFiles();
 473 }
 474
 475 } // namespace
 476
 477 int main(int argc, const char **argv) {
 478   return includeFixerMain(argc, argv);
 479 }