clang-tools-extra/clangd/ParsedAST.cpp

   1 //===--- ParsedAST.cpp -------------------------------------------*- C++-*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "ParsedAST.h"
  10 #include "../clang-tidy/ClangTidyCheck.h"
  11 #include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
  12 #include "../clang-tidy/ClangTidyModule.h"
  13 #include "../clang-tidy/ClangTidyModuleRegistry.h"
  14 #include "../clang-tidy/ClangTidyOptions.h"
  15 #include "AST.h"
  16 #include "CollectMacros.h"
  17 #include "Compiler.h"
  18 #include "Config.h"
  19 #include "Diagnostics.h"
  20 #include "Feature.h"
  21 #include "FeatureModule.h"
  22 #include "Headers.h"
  23 #include "HeuristicResolver.h"
  24 #include "IncludeCleaner.h"
  25 #include "IncludeFixer.h"
  26 #include "Preamble.h"
  27 #include "SourceCode.h"
  28 #include "TidyProvider.h"
  29 #include "clang-include-cleaner/Record.h"
  30 #include "index/Symbol.h"
  31 #include "support/Logger.h"
  32 #include "support/Path.h"
  33 #include "support/Trace.h"
  34 #include "clang/AST/ASTContext.h"
  35 #include "clang/AST/Decl.h"
  36 #include "clang/AST/DeclGroup.h"
  37 #include "clang/AST/ExternalASTSource.h"
  38 #include "clang/ASTMatchers/ASTMatchFinder.h"
  39 #include "clang/Basic/Diagnostic.h"
  40 #include "clang/Basic/DiagnosticIDs.h"
  41 #include "clang/Basic/DiagnosticSema.h"
  42 #include "clang/Basic/FileEntry.h"
  43 #include "clang/Basic/LLVM.h"
  44 #include "clang/Basic/LangOptions.h"
  45 #include "clang/Basic/SourceLocation.h"
  46 #include "clang/Basic/SourceManager.h"
  47 #include "clang/Basic/TokenKinds.h"
  48 #include "clang/Frontend/CompilerInstance.h"
  49 #include "clang/Frontend/CompilerInvocation.h"
  50 #include "clang/Frontend/FrontendActions.h"
  51 #include "clang/Frontend/FrontendOptions.h"
  52 #include "clang/Frontend/PrecompiledPreamble.h"
  53 #include "clang/Lex/Lexer.h"
  54 #include "clang/Lex/PPCallbacks.h"
  55 #include "clang/Lex/Preprocessor.h"
  56 #include "clang/Serialization/ASTWriter.h"
  57 #include "clang/Tooling/CompilationDatabase.h"
  58 #include "clang/Tooling/Core/Diagnostic.h"
  59 #include "clang/Tooling/Syntax/Tokens.h"
  60 #include "llvm/ADT/ArrayRef.h"
  61 #include "llvm/ADT/DenseMap.h"
  62 #include "llvm/ADT/DenseSet.h"
  63 #include "llvm/ADT/STLExtras.h"
  64 #include "llvm/ADT/STLFunctionalExtras.h"
  65 #include "llvm/ADT/SmallVector.h"
  66 #include "llvm/ADT/StringRef.h"
  67 #include "llvm/Support/Error.h"
  68 #include "llvm/Support/MemoryBuffer.h"
  69 #include <cassert>
  70 #include <cstddef>
  71 #include <iterator>
  72 #include <memory>
  73 #include <optional>
  74 #include <string>
  75 #include <tuple>
  76 #include <utility>
  77 #include <vector>
  78
  79 // Force the linker to link in Clang-tidy modules.
  80 // clangd doesn't support the static analyzer.
  81 #if CLANGD_TIDY_CHECKS
  82 #define CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS
  83 #include "../clang-tidy/ClangTidyForceLinker.h"
  84 #endif
  85
  86 namespace clang {
  87 namespace clangd {
  88 namespace {
  89
  90 template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
  91   return Vec.capacity() * sizeof(T);
  92 }
  93
  94 class DeclTrackingASTConsumer : public ASTConsumer {
  95 public:
  96   DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
  97       : TopLevelDecls(TopLevelDecls) {}
  98
  99   bool HandleTopLevelDecl(DeclGroupRef DG) override {
 100     for (Decl *D : DG) {
 101       auto &SM = D->getASTContext().getSourceManager();
 102       if (!isInsideMainFile(D->getLocation(), SM))
 103         continue;
 104       if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
 105         if (isImplicitTemplateInstantiation(ND))
 106           continue;
 107
 108       // ObjCMethodDecl are not actually top-level decls.
 109       if (isa<ObjCMethodDecl>(D))
 110         continue;
 111
 112       TopLevelDecls.push_back(D);
 113     }
 114     return true;
 115   }
 116
 117 private:
 118   std::vector<Decl *> &TopLevelDecls;
 119 };
 120
 121 class ClangdFrontendAction : public SyntaxOnlyAction {
 122 public:
 123   std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
 124
 125 protected:
 126   std::unique_ptr<ASTConsumer>
 127   CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
 128     return std::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
 129   }
 130
 131 private:
 132   std::vector<Decl *> TopLevelDecls;
 133 };
 134
 135 // When using a preamble, only preprocessor events outside its bounds are seen.
 136 // This is almost what we want: replaying transitive preprocessing wastes time.
 137 // However this confuses clang-tidy checks: they don't see any #includes!
 138 // So we replay the *non-transitive* #includes that appear in the main-file.
 139 // It would be nice to replay other events (macro definitions, ifdefs etc) but
 140 // this addresses the most common cases fairly cheaply.
 141 class ReplayPreamble : private PPCallbacks {
 142 public:
 143   // Attach preprocessor hooks such that preamble events will be injected at
 144   // the appropriate time.
 145   // Events will be delivered to the *currently registered* PP callbacks.
 146   static void attach(std::vector<Inclusion> Includes, CompilerInstance &Clang,
 147                      const PreambleBounds &PB) {
 148     auto &PP = Clang.getPreprocessor();
 149     auto *ExistingCallbacks = PP.getPPCallbacks();
 150     // No need to replay events if nobody is listening.
 151     if (!ExistingCallbacks)
 152       return;
 153     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(new ReplayPreamble(
 154         std::move(Includes), ExistingCallbacks, Clang.getSourceManager(), PP,
 155         Clang.getLangOpts(), PB)));
 156     // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
 157     // around, creating a chaining wrapper. Guard against other implementations.
 158     assert(PP.getPPCallbacks() != ExistingCallbacks &&
 159            "Expected chaining implementation");
 160   }
 161
 162 private:
 163   ReplayPreamble(std::vector<Inclusion> Includes, PPCallbacks *Delegate,
 164                  const SourceManager &SM, Preprocessor &PP,
 165                  const LangOptions &LangOpts, const PreambleBounds &PB)
 166       : Includes(std::move(Includes)), Delegate(Delegate), SM(SM), PP(PP) {
 167     // Only tokenize the preamble section of the main file, as we are not
 168     // interested in the rest of the tokens.
 169     MainFileTokens = syntax::tokenize(
 170         syntax::FileRange(SM.getMainFileID(), 0, PB.Size), SM, LangOpts);
 171   }
 172
 173   // In a normal compile, the preamble traverses the following structure:
 174   //
 175   // mainfile.cpp
 176   //   <built-in>
 177   //     ... macro definitions like __cplusplus ...
 178   //     <command-line>
 179   //       ... macro definitions for args like -Dfoo=bar ...
 180   //   "header1.h"
 181   //     ... header file contents ...
 182   //   "header2.h"
 183   //     ... header file contents ...
 184   //   ... main file contents ...
 185   //
 186   // When using a preamble, the "header1" and "header2" subtrees get skipped.
 187   // We insert them right after the built-in header, which still appears.
 188   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
 189                    SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
 190     // It'd be nice if there was a better way to identify built-in headers...
 191     if (Reason == FileChangeReason::ExitFile &&
 192         SM.getBufferOrFake(PrevFID).getBufferIdentifier() == "<built-in>")
 193       replay();
 194   }
 195
 196   void replay() {
 197     for (const auto &Inc : Includes) {
 198       OptionalFileEntryRef File;
 199       if (Inc.Resolved != "")
 200         File = expectedToOptional(SM.getFileManager().getFileRef(Inc.Resolved));
 201
 202       // Re-lex the #include directive to find its interesting parts.
 203       auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset);
 204       auto HashTok = llvm::partition_point(MainFileTokens,
 205                                            [&HashLoc](const syntax::Token &T) {
 206                                              return T.location() < HashLoc;
 207                                            });
 208       assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash);
 209
 210       auto IncludeTok = std::next(HashTok);
 211       assert(IncludeTok != MainFileTokens.end());
 212
 213       auto FileTok = std::next(IncludeTok);
 214       assert(FileTok != MainFileTokens.end());
 215
 216       // Create a fake import/include token, none of the callers seem to care
 217       // about clang::Token::Flags.
 218       Token SynthesizedIncludeTok;
 219       SynthesizedIncludeTok.startToken();
 220       SynthesizedIncludeTok.setLocation(IncludeTok->location());
 221       SynthesizedIncludeTok.setLength(IncludeTok->length());
 222       SynthesizedIncludeTok.setKind(tok::raw_identifier);
 223       SynthesizedIncludeTok.setRawIdentifierData(IncludeTok->text(SM).data());
 224       PP.LookUpIdentifierInfo(SynthesizedIncludeTok);
 225
 226       // Same here, create a fake one for Filename, including angles or quotes.
 227       Token SynthesizedFilenameTok;
 228       SynthesizedFilenameTok.startToken();
 229       SynthesizedFilenameTok.setLocation(FileTok->location());
 230       // Note that we can't make use of FileTok->length/text in here as in the
 231       // case of angled includes this will contain tok::less instead of
 232       // filename. Whereas Inc.Written contains the full header name including
 233       // quotes/angles.
 234       SynthesizedFilenameTok.setLength(Inc.Written.length());
 235       SynthesizedFilenameTok.setKind(tok::header_name);
 236       SynthesizedFilenameTok.setLiteralData(Inc.Written.data());
 237
 238       llvm::StringRef WrittenFilename =
 239           llvm::StringRef(Inc.Written).drop_front().drop_back();
 240       Delegate->InclusionDirective(
 241           HashTok->location(), SynthesizedIncludeTok, WrittenFilename,
 242           Inc.Written.front() == '<',
 243           syntax::FileRange(SM, SynthesizedFilenameTok.getLocation(),
 244                             SynthesizedFilenameTok.getEndLoc())
 245               .toCharRange(SM),
 246           File, "SearchPath", "RelPath",
 247           /*SuggestedModule=*/nullptr, /*ModuleImported=*/false, Inc.FileKind);
 248       if (File)
 249         Delegate->FileSkipped(*File, SynthesizedFilenameTok, Inc.FileKind);
 250     }
 251   }
 252
 253   const std::vector<Inclusion> Includes;
 254   PPCallbacks *Delegate;
 255   const SourceManager &SM;
 256   Preprocessor &PP;
 257   std::vector<syntax::Token> MainFileTokens;
 258 };
 259
 260 // Filter for clang diagnostics groups enabled by CTOptions.Checks.
 261 //
 262 // These are check names like clang-diagnostics-unused.
 263 // Note that unlike -Wunused, clang-diagnostics-unused does not imply
 264 // subcategories like clang-diagnostics-unused-function.
 265 //
 266 // This is used to determine which diagnostics can be enabled by ExtraArgs in
 267 // the clang-tidy configuration.
 268 class TidyDiagnosticGroups {
 269   // Whether all diagnostic groups are enabled by default.
 270   // True if we've seen clang-diagnostic-*.
 271   bool Default = false;
 272   // Set of diag::Group whose enablement != Default.
 273   // If Default is false, this is foo where we've seen clang-diagnostic-foo.
 274   llvm::DenseSet<unsigned> Exceptions;
 275
 276 public:
 277   TidyDiagnosticGroups(llvm::StringRef Checks) {
 278     constexpr llvm::StringLiteral CDPrefix = "clang-diagnostic-";
 279
 280     llvm::StringRef Check;
 281     while (!Checks.empty()) {
 282       std::tie(Check, Checks) = Checks.split(',');
 283       Check = Check.trim();
 284
 285       if (Check.empty())
 286         continue;
 287
 288       bool Enable = !Check.consume_front("-");
 289       bool Glob = Check.consume_back("*");
 290       if (Glob) {
 291         // Is this clang-diagnostic-*, or *, or so?
 292         // (We ignore all other types of globs).
 293         if (CDPrefix.starts_with(Check)) {
 294           Default = Enable;
 295           Exceptions.clear();
 296         }
 297         continue;
 298       }
 299
 300       // In "*,clang-diagnostic-foo", the latter is a no-op.
 301       if (Default == Enable)
 302         continue;
 303       // The only non-glob entries we care about are clang-diagnostic-foo.
 304       if (!Check.consume_front(CDPrefix))
 305         continue;
 306
 307       if (auto Group = DiagnosticIDs::getGroupForWarningOption(Check))
 308         Exceptions.insert(static_cast<unsigned>(*Group));
 309     }
 310   }
 311
 312   bool operator()(diag::Group GroupID) const {
 313     return Exceptions.contains(static_cast<unsigned>(GroupID)) ? !Default
 314                                                                : Default;
 315   }
 316 };
 317
 318 // Find -W<group> and -Wno-<group> options in ExtraArgs and apply them to Diags.
 319 //
 320 // This is used to handle ExtraArgs in clang-tidy configuration.
 321 // We don't use clang's standard handling of this as we want slightly different
 322 // behavior (e.g. we want to exclude these from -Wno-error).
 323 void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs,
 324                          llvm::function_ref<bool(diag::Group)> EnabledGroups,
 325                          DiagnosticsEngine &Diags) {
 326   for (llvm::StringRef Group : ExtraArgs) {
 327     // Only handle args that are of the form -W[no-]<group>.
 328     // Other flags are possible but rare and deliberately out of scope.
 329     llvm::SmallVector<diag::kind> Members;
 330     if (!Group.consume_front("-W") || Group.empty())
 331       continue;
 332     bool Enable = !Group.consume_front("no-");
 333     if (Diags.getDiagnosticIDs()->getDiagnosticsInGroup(
 334             diag::Flavor::WarningOrError, Group, Members))
 335       continue;
 336
 337     // Upgrade (or downgrade) the severity of each diagnostic in the group.
 338     // If -Werror is on, newly added warnings will be treated as errors.
 339     // We don't want this, so keep track of them to fix afterwards.
 340     bool NeedsWerrorExclusion = false;
 341     for (diag::kind ID : Members) {
 342       if (Enable) {
 343         if (Diags.getDiagnosticLevel(ID, SourceLocation()) <
 344             DiagnosticsEngine::Warning) {
 345           auto Group = DiagnosticIDs::getGroupForDiag(ID);
 346           if (!Group || !EnabledGroups(*Group))
 347             continue;
 348           Diags.setSeverity(ID, diag::Severity::Warning, SourceLocation());
 349           if (Diags.getWarningsAsErrors())
 350             NeedsWerrorExclusion = true;
 351         }
 352       } else {
 353         Diags.setSeverity(ID, diag::Severity::Ignored, SourceLocation());
 354       }
 355     }
 356     if (NeedsWerrorExclusion) {
 357       // FIXME: there's no API to suppress -Werror for single diagnostics.
 358       // In some cases with sub-groups, we may end up erroneously
 359       // downgrading diagnostics that were -Werror in the compile command.
 360       Diags.setDiagnosticGroupWarningAsError(Group, false);
 361     }
 362   }
 363 }
 364
 365 std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code,
 366                                          const ThreadsafeFS &TFS) {
 367   auto &Cfg = Config::current();
 368   if (Cfg.Diagnostics.SuppressAll)
 369     return {};
 370   bool SuppressMissing =
 371       Cfg.Diagnostics.Suppress.contains("missing-includes") ||
 372       Cfg.Diagnostics.MissingIncludes == Config::IncludesPolicy::None;
 373   bool SuppressUnused =
 374       Cfg.Diagnostics.Suppress.contains("unused-includes") ||
 375       Cfg.Diagnostics.UnusedIncludes == Config::IncludesPolicy::None;
 376   if (SuppressMissing && SuppressUnused)
 377     return {};
 378   auto Findings = computeIncludeCleanerFindings(
 379       AST, Cfg.Diagnostics.Includes.AnalyzeAngledIncludes);
 380   if (SuppressMissing)
 381     Findings.MissingIncludes.clear();
 382   if (SuppressUnused)
 383     Findings.UnusedIncludes.clear();
 384   return issueIncludeCleanerDiagnostics(AST, Code, Findings, TFS,
 385                                         Cfg.Diagnostics.Includes.IgnoreHeader);
 386 }
 387
 388 tidy::ClangTidyCheckFactories
 389 filterFastTidyChecks(const tidy::ClangTidyCheckFactories &All,
 390                      Config::FastCheckPolicy Policy) {
 391   if (Policy == Config::FastCheckPolicy::None)
 392     return All;
 393   bool AllowUnknown = Policy == Config::FastCheckPolicy::Loose;
 394   tidy::ClangTidyCheckFactories Fast;
 395   for (const auto &Factory : All) {
 396     if (isFastTidyCheck(Factory.getKey()).value_or(AllowUnknown))
 397       Fast.registerCheckFactory(Factory.first(), Factory.second);
 398   }
 399   return Fast;
 400 }
 401
 402 } // namespace
 403
 404 std::optional<ParsedAST>
 405 ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
 406                  std::unique_ptr<clang::CompilerInvocation> CI,
 407                  llvm::ArrayRef<Diag> CompilerInvocationDiags,
 408                  std::shared_ptr<const PreambleData> Preamble) {
 409   trace::Span Tracer("BuildAST");
 410   SPAN_ATTACH(Tracer, "File", Filename);
 411   const Config &Cfg = Config::current();
 412
 413   auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
 414   if (Preamble && Preamble->StatCache)
 415     VFS = Preamble->StatCache->getConsumingFS(std::move(VFS));
 416
 417   assert(CI);
 418
 419   if (CI->getFrontendOpts().Inputs.size() > 0) {
 420     auto Lang = CI->getFrontendOpts().Inputs[0].getKind().getLanguage();
 421     if (Lang == Language::Asm || Lang == Language::LLVM_IR) {
 422       elog("Clangd does not support assembly or IR source files");
 423       return std::nullopt;
 424     }
 425   }
 426
 427   // Command-line parsing sets DisableFree to true by default, but we don't want
 428   // to leak memory in clangd.
 429   CI->getFrontendOpts().DisableFree = false;
 430   const PrecompiledPreamble *PreamblePCH =
 431       Preamble ? &Preamble->Preamble : nullptr;
 432
 433   // This is on-by-default in windows to allow parsing SDK headers, but it
 434   // breaks many features. Disable it for the main-file (not preamble).
 435   CI->getLangOpts().DelayedTemplateParsing = false;
 436
 437   std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
 438   if (Inputs.FeatureModules) {
 439     for (auto &M : *Inputs.FeatureModules) {
 440       if (auto Listener = M.astListeners())
 441         ASTListeners.emplace_back(std::move(Listener));
 442     }
 443   }
 444   StoreDiags ASTDiags;
 445   ASTDiags.setDiagCallback(
 446       [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
 447         for (const auto &L : ASTListeners)
 448           L->sawDiagnostic(D, Diag);
 449       });
 450
 451   // Adjust header search options to load the built module files recorded
 452   // in RequiredModules.
 453   if (Preamble && Preamble->RequiredModules)
 454     Preamble->RequiredModules->adjustHeaderSearchOptions(
 455         CI->getHeaderSearchOpts());
 456
 457   std::optional<PreamblePatch> Patch;
 458   // We might use an ignoring diagnostic consumer if they are going to be
 459   // dropped later on to not pay for extra latency by processing them.
 460   DiagnosticConsumer *DiagConsumer = &ASTDiags;
 461   IgnoreDiagnostics DropDiags;
 462   if (Preamble) {
 463     Patch = PreamblePatch::createFullPatch(Filename, Inputs, *Preamble);
 464     Patch->apply(*CI);
 465   }
 466   auto Clang = prepareCompilerInstance(
 467       std::move(CI), PreamblePCH,
 468       llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS,
 469       *DiagConsumer);
 470
 471   if (!Clang) {
 472     // The last diagnostic contains information about the reason of this
 473     // failure.
 474     std::vector<Diag> Diags(ASTDiags.take());
 475     elog("Failed to prepare a compiler instance: {0}",
 476          !Diags.empty() ? static_cast<DiagBase &>(Diags.back()).Message
 477                         : "unknown error");
 478     return std::nullopt;
 479   }
 480   tidy::ClangTidyOptions ClangTidyOpts;
 481   {
 482     trace::Span Tracer("ClangTidyOpts");
 483     ClangTidyOpts = getTidyOptionsForFile(Inputs.ClangTidyProvider, Filename);
 484     dlog("ClangTidy configuration for file {0}: {1}", Filename,
 485          tidy::configurationAsText(ClangTidyOpts));
 486
 487     // If clang-tidy is configured to emit clang warnings, we should too.
 488     //
 489     // Such clang-tidy configuration consists of two parts:
 490     //   - ExtraArgs: ["-Wfoo"] causes clang to produce the warnings
 491     //   - Checks: "clang-diagnostic-foo" prevents clang-tidy filtering them out
 492     //
 493     // In clang-tidy, diagnostics are emitted if they pass both checks.
 494     // When groups contain subgroups, -Wparent includes the child, but
 495     // clang-diagnostic-parent does not.
 496     //
 497     // We *don't* want to change the compile command directly. This can have
 498     // too many unexpected effects: breaking the command, interactions with
 499     // -- and -Werror, etc. Besides, we've already parsed the command.
 500     // Instead we parse the -W<group> flags and handle them directly.
 501     //
 502     // Similarly, we don't want to use Checks to filter clang diagnostics after
 503     // they are generated, as this spreads clang-tidy emulation everywhere.
 504     // Instead, we just use these to filter which extra diagnostics we enable.
 505     auto &Diags = Clang->getDiagnostics();
 506     TidyDiagnosticGroups TidyGroups(ClangTidyOpts.Checks ? *ClangTidyOpts.Checks
 507                                                          : llvm::StringRef());
 508     if (ClangTidyOpts.ExtraArgsBefore)
 509       applyWarningOptions(*ClangTidyOpts.ExtraArgsBefore, TidyGroups, Diags);
 510     if (ClangTidyOpts.ExtraArgs)
 511       applyWarningOptions(*ClangTidyOpts.ExtraArgs, TidyGroups, Diags);
 512   }
 513
 514   auto Action = std::make_unique<ClangdFrontendAction>();
 515   const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
 516   if (!Action->BeginSourceFile(*Clang, MainInput)) {
 517     elog("BeginSourceFile() failed when building AST for {0}",
 518          MainInput.getFile());
 519     return std::nullopt;
 520   }
 521   // If we saw an include guard in the preamble section of the main file,
 522   // mark the main-file as include-guarded.
 523   // This information is part of the HeaderFileInfo but is not loaded from the
 524   // preamble as the file's size is part of its identity and may have changed.
 525   // (The rest of HeaderFileInfo is not relevant for our purposes).
 526   if (Preamble && Preamble->MainIsIncludeGuarded) {
 527     const SourceManager &SM = Clang->getSourceManager();
 528     OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(SM.getMainFileID());
 529     Clang->getPreprocessor().getHeaderSearchInfo().MarkFileIncludeOnce(*MainFE);
 530   }
 531
 532   // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
 533   // Clang-tidy has some limitations to ensure reasonable performance:
 534   //  - checks don't see all preprocessor events in the preamble
 535   //  - matchers run only over the main-file top-level decls (and can't see
 536   //    ancestors outside this scope).
 537   // In practice almost all checks work well without modifications.
 538   std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
 539   ast_matchers::MatchFinder CTFinder;
 540   std::optional<tidy::ClangTidyContext> CTContext;
 541   // Must outlive FixIncludes.
 542   auto BuildDir = VFS->getCurrentWorkingDirectory();
 543   std::optional<IncludeFixer> FixIncludes;
 544   llvm::DenseMap<diag::kind, DiagnosticsEngine::Level> OverriddenSeverity;
 545   // No need to run clang-tidy or IncludeFixerif we are not going to surface
 546   // diagnostics.
 547   {
 548     trace::Span Tracer("ClangTidyInit");
 549     static const auto *AllCTFactories = [] {
 550       auto *CTFactories = new tidy::ClangTidyCheckFactories;
 551       for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
 552         E.instantiate()->addCheckFactories(*CTFactories);
 553       return CTFactories;
 554     }();
 555     tidy::ClangTidyCheckFactories FastFactories = filterFastTidyChecks(
 556         *AllCTFactories, Cfg.Diagnostics.ClangTidy.FastCheckFilter);
 557     CTContext.emplace(std::make_unique<tidy::DefaultOptionsProvider>(
 558         tidy::ClangTidyGlobalOptions(), ClangTidyOpts));
 559     CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
 560     CTContext->setASTContext(&Clang->getASTContext());
 561     CTContext->setCurrentFile(Filename);
 562     CTContext->setSelfContainedDiags(true);
 563     CTChecks = FastFactories.createChecksForLanguage(&*CTContext);
 564     Preprocessor *PP = &Clang->getPreprocessor();
 565     for (const auto &Check : CTChecks) {
 566       Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
 567       Check->registerMatchers(&CTFinder);
 568     }
 569
 570     // Clang only corrects typos for use of undeclared functions in C if that
 571     // use is an error. Include fixer relies on typo correction, so pretend
 572     // this is an error. (The actual typo correction is nice too).
 573     // We restore the original severity in the level adjuster.
 574     // FIXME: It would be better to have a real API for this, but what?
 575     for (auto ID : {diag::ext_implicit_function_decl_c99,
 576                     diag::ext_implicit_lib_function_decl,
 577                     diag::ext_implicit_lib_function_decl_c99,
 578                     diag::warn_implicit_function_decl}) {
 579       OverriddenSeverity.try_emplace(
 580           ID, Clang->getDiagnostics().getDiagnosticLevel(ID, SourceLocation()));
 581       Clang->getDiagnostics().setSeverity(ID, diag::Severity::Error,
 582                                           SourceLocation());
 583     }
 584
 585     ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
 586                                   const clang::Diagnostic &Info) {
 587       if (Cfg.Diagnostics.SuppressAll ||
 588           isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
 589                                         Clang->getLangOpts()))
 590         return DiagnosticsEngine::Ignored;
 591
 592       auto It = OverriddenSeverity.find(Info.getID());
 593       if (It != OverriddenSeverity.end())
 594         DiagLevel = It->second;
 595
 596       if (!CTChecks.empty()) {
 597         std::string CheckName = CTContext->getCheckName(Info.getID());
 598         bool IsClangTidyDiag = !CheckName.empty();
 599         if (IsClangTidyDiag) {
 600           if (Cfg.Diagnostics.Suppress.contains(CheckName))
 601             return DiagnosticsEngine::Ignored;
 602           // Check for suppression comment. Skip the check for diagnostics not
 603           // in the main file, because we don't want that function to query the
 604           // source buffer for preamble files. For the same reason, we ask
 605           // shouldSuppressDiagnostic to avoid I/O.
 606           // We let suppression comments take precedence over warning-as-error
 607           // to match clang-tidy's behaviour.
 608           bool IsInsideMainFile =
 609               Info.hasSourceManager() &&
 610               isInsideMainFile(Info.getLocation(), Info.getSourceManager());
 611           SmallVector<tooling::Diagnostic, 1> TidySuppressedErrors;
 612           if (IsInsideMainFile && CTContext->shouldSuppressDiagnostic(
 613                                       DiagLevel, Info, TidySuppressedErrors,
 614                                       /*AllowIO=*/false,
 615                                       /*EnableNolintBlocks=*/true)) {
 616             // FIXME: should we expose the suppression error (invalid use of
 617             // NOLINT comments)?
 618             return DiagnosticsEngine::Ignored;
 619           }
 620           if (!CTContext->getOptions().SystemHeaders.value_or(false) &&
 621               Info.hasSourceManager() &&
 622               Info.getSourceManager().isInSystemMacro(Info.getLocation()))
 623             return DiagnosticsEngine::Ignored;
 624
 625           // Check for warning-as-error.
 626           if (DiagLevel == DiagnosticsEngine::Warning &&
 627               CTContext->treatAsError(CheckName)) {
 628             return DiagnosticsEngine::Error;
 629           }
 630         }
 631       }
 632       return DiagLevel;
 633     });
 634
 635     // Add IncludeFixer which can recover diagnostics caused by missing includes
 636     // (e.g. incomplete type) and attach include insertion fixes to diagnostics.
 637     if (Inputs.Index && !BuildDir.getError()) {
 638       auto Style =
 639           getFormatStyleForFile(Filename, Inputs.Contents, *Inputs.TFS, false);
 640       auto Inserter = std::make_shared<IncludeInserter>(
 641           Filename, Inputs.Contents, Style, BuildDir.get(),
 642           &Clang->getPreprocessor().getHeaderSearchInfo());
 643       ArrayRef<Inclusion> MainFileIncludes;
 644       if (Preamble) {
 645         MainFileIncludes = Preamble->Includes.MainFileIncludes;
 646         for (const auto &Inc : Preamble->Includes.MainFileIncludes)
 647           Inserter->addExisting(Inc);
 648       }
 649       // FIXME: Consider piping through ASTSignals to fetch this to handle the
 650       // case where a header file contains ObjC decls but no #imports.
 651       Symbol::IncludeDirective Directive =
 652           Inputs.Opts.ImportInsertions
 653               ? preferredIncludeDirective(Filename, Clang->getLangOpts(),
 654                                           MainFileIncludes, {})
 655               : Symbol::Include;
 656       FixIncludes.emplace(Filename, Inserter, *Inputs.Index,
 657                           /*IndexRequestLimit=*/5, Directive);
 658       ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl,
 659                                               const clang::Diagnostic &Info) {
 660         return FixIncludes->fix(DiagLevl, Info);
 661       });
 662       Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder());
 663     }
 664   }
 665
 666   IncludeStructure Includes;
 667   include_cleaner::PragmaIncludes PI;
 668   // If we are using a preamble, copy existing includes.
 669   if (Preamble) {
 670     Includes = Preamble->Includes;
 671     Includes.MainFileIncludes = Patch->preambleIncludes();
 672     // Replay the preamble includes so that clang-tidy checks can see them.
 673     ReplayPreamble::attach(Patch->preambleIncludes(), *Clang,
 674                            Patch->modifiedBounds());
 675     PI = *Preamble->Pragmas;
 676   }
 677   // Important: collectIncludeStructure is registered *after* ReplayPreamble!
 678   // Otherwise we would collect the replayed includes again...
 679   // (We can't *just* use the replayed includes, they don't have Resolved path).
 680   Includes.collect(*Clang);
 681   // Same for pragma-includes, we're already inheriting preamble includes, so we
 682   // should only receive callbacks for non-preamble mainfile includes.
 683   PI.record(*Clang);
 684   // Copy over the macros in the preamble region of the main file, and combine
 685   // with non-preamble macros below.
 686   MainFileMacros Macros;
 687   std::vector<PragmaMark> Marks;
 688   if (Preamble) {
 689     Macros = Patch->mainFileMacros();
 690     Marks = Patch->marks();
 691   }
 692   auto &PP = Clang->getPreprocessor();
 693   auto MacroCollector = std::make_unique<CollectMainFileMacros>(PP, Macros);
 694   auto *MacroCollectorPtr = MacroCollector.get(); // so we can call doneParse()
 695   PP.addPPCallbacks(std::move(MacroCollector));
 696
 697   PP.addPPCallbacks(
 698       collectPragmaMarksCallback(Clang->getSourceManager(), Marks));
 699
 700   // FIXME: Attach a comment handler to take care of
 701   // keep/export/no_include etc. IWYU pragmas.
 702
 703   // Collect tokens of the main file.
 704   syntax::TokenCollector CollectTokens(PP);
 705
 706   // To remain consistent with preamble builds, these callbacks must be called
 707   // exactly here, after preprocessor is initialized and BeginSourceFile() was
 708   // called already.
 709   for (const auto &L : ASTListeners)
 710     L->beforeExecute(*Clang);
 711
 712   if (llvm::Error Err = Action->Execute())
 713     log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(),
 714         toString(std::move(Err)));
 715
 716   // Disable the macro collector for the remainder of this function, e.g.
 717   // clang-tidy checkers.
 718   MacroCollectorPtr->doneParse();
 719
 720   // We have to consume the tokens before running clang-tidy to avoid collecting
 721   // tokens from running the preprocessor inside the checks (only
 722   // modernize-use-trailing-return-type does that today).
 723   syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
 724   // Makes SelectionTree build much faster.
 725   Tokens.indexExpandedTokens();
 726   std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
 727   // AST traversals should exclude the preamble, to avoid performance cliffs.
 728   Clang->getASTContext().setTraversalScope(ParsedDecls);
 729   if (!CTChecks.empty()) {
 730     // Run the AST-dependent part of the clang-tidy checks.
 731     // (The preprocessor part ran already, via PPCallbacks).
 732     trace::Span Tracer("ClangTidyMatch");
 733     CTFinder.matchAST(Clang->getASTContext());
 734   }
 735
 736   // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
 737   // However Action->EndSourceFile() would destroy the ASTContext!
 738   // So just inform the preprocessor of EOF, while keeping everything alive.
 739   PP.EndSourceFile();
 740   // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
 741   // has a longer lifetime.
 742   Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
 743   // CompilerInstance won't run this callback, do it directly.
 744   ASTDiags.EndSourceFile();
 745
 746   std::vector<Diag> Diags = CompilerInvocationDiags;
 747   // FIXME: Also skip generation of diagnostics altogether to speed up ast
 748   // builds when we are patching a stale preamble.
 749   // Add diagnostics from the preamble, if any.
 750   if (Preamble)
 751     llvm::append_range(Diags, Patch->patchedDiags());
 752   // Finally, add diagnostics coming from the AST.
 753   {
 754     std::vector<Diag> D = ASTDiags.take(&*CTContext);
 755     Diags.insert(Diags.end(), D.begin(), D.end());
 756   }
 757   ParsedAST Result(Filename, Inputs.Version, std::move(Preamble),
 758                    std::move(Clang), std::move(Action), std::move(Tokens),
 759                    std::move(Macros), std::move(Marks), std::move(ParsedDecls),
 760                    std::move(Diags), std::move(Includes), std::move(PI));
 761   llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents, *Inputs.TFS),
 762              std::back_inserter(Result.Diags));
 763   return std::move(Result);
 764 }
 765
 766 ParsedAST::ParsedAST(ParsedAST &&Other) = default;
 767
 768 ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
 769
 770 ParsedAST::~ParsedAST() {
 771   if (Action) {
 772     // We already notified the PP of end-of-file earlier, so detach it first.
 773     // We must keep it alive until after EndSourceFile(), Sema relies on this.
 774     auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
 775     Clang->setPreprocessor(nullptr);       // Detach so we don't send EOF again.
 776     Action->EndSourceFile();               // Destroy ASTContext and Sema.
 777     // Now Sema is gone, it's safe for PP to go out of scope.
 778   }
 779 }
 780
 781 ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
 782
 783 const ASTContext &ParsedAST::getASTContext() const {
 784   return Clang->getASTContext();
 785 }
 786
 787 Sema &ParsedAST::getSema() { return Clang->getSema(); }
 788
 789 Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
 790
 791 std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
 792   return Clang->getPreprocessorPtr();
 793 }
 794
 795 const Preprocessor &ParsedAST::getPreprocessor() const {
 796   return Clang->getPreprocessor();
 797 }
 798
 799 llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
 800   return LocalTopLevelDecls;
 801 }
 802
 803 llvm::ArrayRef<const Decl *> ParsedAST::getLocalTopLevelDecls() const {
 804   return LocalTopLevelDecls;
 805 }
 806
 807 const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
 808 const std::vector<PragmaMark> &ParsedAST::getMarks() const { return Marks; }
 809
 810 std::size_t ParsedAST::getUsedBytes() const {
 811   auto &AST = getASTContext();
 812   // FIXME(ibiryukov): we do not account for the dynamically allocated part of
 813   // Message and Fixes inside each diagnostic.
 814   std::size_t Total =
 815       clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags);
 816
 817   // FIXME: the rest of the function is almost a direct copy-paste from
 818   // libclang's clang_getCXTUResourceUsage. We could share the implementation.
 819
 820   // Sum up various allocators inside the ast context and the preprocessor.
 821   Total += AST.getASTAllocatedMemory();
 822   Total += AST.getSideTableAllocatedMemory();
 823   Total += AST.Idents.getAllocator().getTotalMemory();
 824   Total += AST.Selectors.getTotalMemory();
 825
 826   Total += AST.getSourceManager().getContentCacheSize();
 827   Total += AST.getSourceManager().getDataStructureSizes();
 828   Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
 829
 830   if (ExternalASTSource *Ext = AST.getExternalSource())
 831     Total += Ext->getMemoryBufferSizes().malloc_bytes;
 832
 833   const Preprocessor &PP = getPreprocessor();
 834   Total += PP.getTotalMemory();
 835   if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
 836     Total += PRec->getTotalMemory();
 837   Total += PP.getHeaderSearchInfo().getTotalMemory();
 838
 839   return Total;
 840 }
 841
 842 const IncludeStructure &ParsedAST::getIncludeStructure() const {
 843   return Includes;
 844 }
 845
 846 ParsedAST::ParsedAST(PathRef TUPath, llvm::StringRef Version,
 847                      std::shared_ptr<const PreambleData> Preamble,
 848                      std::unique_ptr<CompilerInstance> Clang,
 849                      std::unique_ptr<FrontendAction> Action,
 850                      syntax::TokenBuffer Tokens, MainFileMacros Macros,
 851                      std::vector<PragmaMark> Marks,
 852                      std::vector<Decl *> LocalTopLevelDecls,
 853                      std::vector<Diag> Diags, IncludeStructure Includes,
 854                      include_cleaner::PragmaIncludes PI)
 855     : TUPath(TUPath), Version(Version), Preamble(std::move(Preamble)),
 856       Clang(std::move(Clang)), Action(std::move(Action)),
 857       Tokens(std::move(Tokens)), Macros(std::move(Macros)),
 858       Marks(std::move(Marks)), Diags(std::move(Diags)),
 859       LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
 860       Includes(std::move(Includes)), PI(std::move(PI)),
 861       Resolver(std::make_unique<HeuristicResolver>(getASTContext())) {
 862   assert(this->Clang);
 863   assert(this->Action);
 864 }
 865
 866 const include_cleaner::PragmaIncludes &ParsedAST::getPragmaIncludes() const {
 867   return PI;
 868 }
 869
 870 std::optional<llvm::StringRef> ParsedAST::preambleVersion() const {
 871   if (!Preamble)
 872     return std::nullopt;
 873   return llvm::StringRef(Preamble->Version);
 874 }
 875
 876 llvm::ArrayRef<Diag> ParsedAST::getDiagnostics() const { return Diags; }
 877 } // namespace clangd
 878 } // namespace clang