clang-tools-extra/clangd/Preamble.h

   1 //===--- Preamble.h - Reusing expensive parts of the AST ---------*- C++-*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // The vast majority of code in a typical translation unit is in the headers
  10 // included at the top of the file.
  11 //
  12 // The preamble optimization says that we can parse this code once, and reuse
  13 // the result multiple times. The preamble is invalidated by changes to the
  14 // code in the preamble region, to the compile command, or to files on disk.
  15 //
  16 // This is the most important optimization in clangd: it allows operations like
  17 // code-completion to have sub-second latency. It is supported by the
  18 // PrecompiledPreamble functionality in clang, which wraps the techniques used
  19 // by PCH files, modules etc into a convenient interface.
  20 //
  21 //===----------------------------------------------------------------------===//
  22 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
  23 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
  24
  25 #include "CollectMacros.h"
  26 #include "Compiler.h"
  27 #include "Diagnostics.h"
  28 #include "FS.h"
  29 #include "Headers.h"
  30 #include "ModulesBuilder.h"
  31
  32 #include "clang-include-cleaner/Record.h"
  33 #include "support/Path.h"
  34 #include "clang/Basic/SourceManager.h"
  35 #include "clang/Basic/TargetOptions.h"
  36 #include "clang/Frontend/CompilerInvocation.h"
  37 #include "clang/Frontend/PrecompiledPreamble.h"
  38 #include "clang/Lex/Lexer.h"
  39 #include "clang/Tooling/CompilationDatabase.h"
  40 #include "llvm/ADT/ArrayRef.h"
  41 #include "llvm/ADT/StringRef.h"
  42
  43 #include <cstddef>
  44 #include <functional>
  45 #include <memory>
  46 #include <string>
  47 #include <utility>
  48 #include <vector>
  49
  50 namespace clang {
  51 namespace clangd {
  52
  53 /// The captured AST context.
  54 /// Keeps necessary structs for an ASTContext and Preprocessor alive.
  55 /// This enables consuming them after context that produced the AST is gone.
  56 /// (e.g. indexing a preamble ast on a separate thread). ASTContext stored
  57 /// inside is still not thread-safe.
  58
  59 struct CapturedASTCtx {
  60 public:
  61   CapturedASTCtx(CompilerInstance &Clang)
  62       : Invocation(Clang.getInvocationPtr()),
  63         Diagnostics(Clang.getDiagnosticsPtr()), Target(Clang.getTargetPtr()),
  64         AuxTarget(Clang.getAuxTarget()), FileMgr(Clang.getFileManagerPtr()),
  65         SourceMgr(Clang.getSourceManagerPtr()), PP(Clang.getPreprocessorPtr()),
  66         Context(Clang.getASTContextPtr()) {}
  67
  68   CapturedASTCtx(const CapturedASTCtx &) = delete;
  69   CapturedASTCtx &operator=(const CapturedASTCtx &) = delete;
  70   CapturedASTCtx(CapturedASTCtx &&) = default;
  71   CapturedASTCtx &operator=(CapturedASTCtx &&) = default;
  72
  73   ASTContext &getASTContext() { return *Context; }
  74   Preprocessor &getPreprocessor() { return *PP; }
  75   CompilerInvocation &getCompilerInvocation() { return *Invocation; }
  76   FileManager &getFileManager() { return *FileMgr; }
  77   void setStatCache(std::shared_ptr<PreambleFileStatusCache> StatCache) {
  78     this->StatCache = StatCache;
  79   }
  80
  81 private:
  82   std::shared_ptr<CompilerInvocation> Invocation;
  83   IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
  84   IntrusiveRefCntPtr<TargetInfo> Target;
  85   IntrusiveRefCntPtr<TargetInfo> AuxTarget;
  86   IntrusiveRefCntPtr<FileManager> FileMgr;
  87   IntrusiveRefCntPtr<SourceManager> SourceMgr;
  88   std::shared_ptr<Preprocessor> PP;
  89   IntrusiveRefCntPtr<ASTContext> Context;
  90   std::shared_ptr<PreambleFileStatusCache> StatCache;
  91 };
  92
  93 /// The parsed preamble and associated data.
  94 ///
  95 /// As we must avoid re-parsing the preamble, any information that can only
  96 /// be obtained during parsing must be eagerly captured and stored here.
  97 struct PreambleData {
  98   PreambleData(PrecompiledPreamble Preamble) : Preamble(std::move(Preamble)) {}
  99
 100   // Version of the ParseInputs this preamble was built from.
 101   std::string Version;
 102   tooling::CompileCommand CompileCommand;
 103   // Target options used when building the preamble. Changes in target can cause
 104   // crashes when deserializing preamble, this enables consumers to use the
 105   // same target (without reparsing CompileCommand).
 106   std::shared_ptr<TargetOptions> TargetOpts = nullptr;
 107   PrecompiledPreamble Preamble;
 108   std::vector<Diag> Diags;
 109   // Processes like code completions and go-to-definitions will need #include
 110   // information, and their compile action skips preamble range.
 111   IncludeStructure Includes;
 112   // Captures #include-mapping information in #included headers.
 113   std::shared_ptr<const include_cleaner::PragmaIncludes> Pragmas;
 114   // Information about required module files for this preamble.
 115   std::unique_ptr<PrerequisiteModules> RequiredModules;
 116   // Macros defined in the preamble section of the main file.
 117   // Users care about headers vs main-file, not preamble vs non-preamble.
 118   // These should be treated as main-file entities e.g. for code completion.
 119   MainFileMacros Macros;
 120   // Pragma marks defined in the preamble section of the main file.
 121   std::vector<PragmaMark> Marks;
 122   // Cache of FS operations performed when building the preamble.
 123   // When reusing a preamble, this cache can be consumed to save IO.
 124   std::shared_ptr<PreambleFileStatusCache> StatCache;
 125   // Whether there was a (possibly-incomplete) include-guard on the main file.
 126   // We need to propagate this information "by hand" to subsequent parses.
 127   bool MainIsIncludeGuarded = false;
 128 };
 129
 130 using PreambleParsedCallback =
 131     std::function<void(CapturedASTCtx ASTCtx,
 132                        std::shared_ptr<const include_cleaner::PragmaIncludes>)>;
 133
 134 /// Timings and statistics from the premble build. Unlike PreambleData, these
 135 /// do not need to be stored for later, but can be useful for logging, metrics,
 136 /// etc.
 137 struct PreambleBuildStats {
 138   /// Total wall time it took to build preamble, in seconds.
 139   double TotalBuildTime;
 140   /// Time spent in filesystem operations during the build, in seconds.
 141   double FileSystemTime;
 142
 143   /// Estimate of the memory used while building the preamble.
 144   /// This memory has been released when buildPreamble returns.
 145   /// For example, this includes the size of the in-memory AST (ASTContext).
 146   size_t BuildSize;
 147   /// The serialized size of the preamble.
 148   /// This storage is needed while the preamble is used (but may be on disk).
 149   size_t SerializedSize;
 150 };
 151
 152 /// Build a preamble for the new inputs unless an old one can be reused.
 153 /// If \p PreambleCallback is set, it will be run on top of the AST while
 154 /// building the preamble.
 155 /// If Stats is not non-null, build statistics will be exported there.
 156 std::shared_ptr<const PreambleData>
 157 buildPreamble(PathRef FileName, CompilerInvocation CI,
 158               const ParseInputs &Inputs, bool StoreInMemory,
 159               PreambleParsedCallback PreambleCallback,
 160               PreambleBuildStats *Stats = nullptr);
 161
 162 /// Returns true if \p Preamble is reusable for \p Inputs. Note that it will
 163 /// return true when some missing headers are now available.
 164 /// FIXME: Should return more information about the delta between \p Preamble
 165 /// and \p Inputs, e.g. new headers.
 166 bool isPreambleCompatible(const PreambleData &Preamble,
 167                           const ParseInputs &Inputs, PathRef FileName,
 168                           const CompilerInvocation &CI);
 169
 170 /// Stores information required to parse a TU using a (possibly stale) Baseline
 171 /// preamble. Later on this information can be injected into the main file by
 172 /// updating compiler invocation with \c apply. This injected section
 173 /// approximately reflects additions to the preamble in Modified contents, e.g.
 174 /// new include directives.
 175 class PreamblePatch {
 176 public:
 177   enum class PatchType { MacroDirectives, All };
 178   /// \p Preamble is used verbatim.
 179   static PreamblePatch unmodified(const PreambleData &Preamble);
 180   /// Builds a patch that contains new PP directives introduced to the preamble
 181   /// section of \p Modified compared to \p Baseline.
 182   /// FIXME: This only handles include directives, we should at least handle
 183   /// define/undef.
 184   static PreamblePatch createFullPatch(llvm::StringRef FileName,
 185                                        const ParseInputs &Modified,
 186                                        const PreambleData &Baseline);
 187   static PreamblePatch createMacroPatch(llvm::StringRef FileName,
 188                                         const ParseInputs &Modified,
 189                                         const PreambleData &Baseline);
 190   /// Returns the FileEntry for the preamble patch of MainFilePath in SM, if
 191   /// any.
 192   static OptionalFileEntryRef getPatchEntry(llvm::StringRef MainFilePath,
 193                                             const SourceManager &SM);
 194
 195   /// Adjusts CI (which compiles the modified inputs) to be used with the
 196   /// baseline preamble. This is done by inserting an artificial include to the
 197   /// \p CI that contains new directives calculated in create.
 198   void apply(CompilerInvocation &CI) const;
 199
 200   /// Returns #include directives from the \c Modified preamble that were
 201   /// resolved using the \c Baseline preamble. This covers the new locations of
 202   /// inclusions that were moved around, but not inclusions of new files. Those
 203   /// will be recorded when parsing the main file: the includes in the injected
 204   /// section will be resolved back to their spelled positions in the main file
 205   /// using the presumed-location mechanism.
 206   std::vector<Inclusion> preambleIncludes() const;
 207
 208   /// Returns preamble bounds for the Modified.
 209   PreambleBounds modifiedBounds() const { return ModifiedBounds; }
 210
 211   /// Returns textual patch contents.
 212   llvm::StringRef text() const { return PatchContents; }
 213
 214   /// Returns diag locations for Modified contents.
 215   llvm::ArrayRef<Diag> patchedDiags() const { return PatchedDiags; }
 216
 217   static constexpr llvm::StringLiteral HeaderName = "__preamble_patch__.h";
 218
 219   llvm::ArrayRef<PragmaMark> marks() const;
 220   const MainFileMacros &mainFileMacros() const;
 221
 222 private:
 223   static PreamblePatch create(llvm::StringRef FileName,
 224                               const ParseInputs &Modified,
 225                               const PreambleData &Baseline,
 226                               PatchType PatchType);
 227
 228   PreamblePatch() = default;
 229   std::string PatchContents;
 230   std::string PatchFileName;
 231   // Includes that are present in both Baseline and Modified. Used for
 232   // patching includes of baseline preamble.
 233   std::vector<Inclusion> PreambleIncludes;
 234   // Diags that were attached to a line preserved in Modified contents.
 235   std::vector<Diag> PatchedDiags;
 236   PreambleBounds ModifiedBounds = {0, false};
 237   const PreambleData *Baseline = nullptr;
 238   std::vector<PragmaMark> PatchedMarks;
 239   MainFileMacros PatchedMacros;
 240 };
 241
 242 } // namespace clangd
 243 } // namespace clang
 244
 245 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H