clang/lib/Interpreter/IncrementalParser.cpp

   1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the class which performs incremental code compilation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "IncrementalParser.h"
  14
  15 #include "clang/AST/DeclContextInternals.h"
  16 #include "clang/CodeGen/BackendUtil.h"
  17 #include "clang/CodeGen/CodeGenAction.h"
  18 #include "clang/CodeGen/ModuleBuilder.h"
  19 #include "clang/Frontend/CompilerInstance.h"
  20 #include "clang/Frontend/FrontendAction.h"
  21 #include "clang/FrontendTool/Utils.h"
  22 #include "clang/Interpreter/Interpreter.h"
  23 #include "clang/Parse/Parser.h"
  24 #include "clang/Sema/Sema.h"
  25 #include "llvm/Option/ArgList.h"
  26 #include "llvm/Support/CrashRecoveryContext.h"
  27 #include "llvm/Support/Error.h"
  28 #include "llvm/Support/Timer.h"
  29
  30 #include <sstream>
  31
  32 namespace clang {
  33
  34 class IncrementalASTConsumer final : public ASTConsumer {
  35   Interpreter &Interp;
  36   std::unique_ptr<ASTConsumer> Consumer;
  37
  38 public:
  39   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
  40       : Interp(InterpRef), Consumer(std::move(C)) {}
  41
  42   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
  43     if (DGR.isNull())
  44       return true;
  45     if (!Consumer)
  46       return true;
  47
  48     for (Decl *D : DGR)
  49       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
  50           TSD && TSD->isSemiMissing())
  51         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
  52
  53     return Consumer->HandleTopLevelDecl(DGR);
  54   }
  55   void HandleTranslationUnit(ASTContext &Ctx) override final {
  56     Consumer->HandleTranslationUnit(Ctx);
  57   }
  58   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
  59     Consumer->HandleInlineFunctionDefinition(D);
  60   }
  61   void HandleInterestingDecl(DeclGroupRef D) override final {
  62     Consumer->HandleInterestingDecl(D);
  63   }
  64   void HandleTagDeclDefinition(TagDecl *D) override final {
  65     Consumer->HandleTagDeclDefinition(D);
  66   }
  67   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
  68     Consumer->HandleTagDeclRequiredDefinition(D);
  69   }
  70   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
  71     Consumer->HandleCXXImplicitFunctionInstantiation(D);
  72   }
  73   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
  74     Consumer->HandleTopLevelDeclInObjCContainer(D);
  75   }
  76   void HandleImplicitImportDecl(ImportDecl *D) override final {
  77     Consumer->HandleImplicitImportDecl(D);
  78   }
  79   void CompleteTentativeDefinition(VarDecl *D) override final {
  80     Consumer->CompleteTentativeDefinition(D);
  81   }
  82   void CompleteExternalDeclaration(DeclaratorDecl *D) override final {
  83     Consumer->CompleteExternalDeclaration(D);
  84   }
  85   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
  86     Consumer->AssignInheritanceModel(RD);
  87   }
  88   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
  89     Consumer->HandleCXXStaticMemberVarInstantiation(D);
  90   }
  91   void HandleVTable(CXXRecordDecl *RD) override final {
  92     Consumer->HandleVTable(RD);
  93   }
  94   ASTMutationListener *GetASTMutationListener() override final {
  95     return Consumer->GetASTMutationListener();
  96   }
  97   ASTDeserializationListener *GetASTDeserializationListener() override final {
  98     return Consumer->GetASTDeserializationListener();
  99   }
 100   void PrintStats() override final { Consumer->PrintStats(); }
 101   bool shouldSkipFunctionBody(Decl *D) override final {
 102     return Consumer->shouldSkipFunctionBody(D);
 103   }
 104   static bool classof(const clang::ASTConsumer *) { return true; }
 105 };
 106
 107 /// A custom action enabling the incremental processing functionality.
 108 ///
 109 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
 110 /// sees a call to \p EndSourceFile it deletes some of the important objects
 111 /// such as \p Preprocessor and \p Sema assuming no further input will come.
 112 ///
 113 /// \p IncrementalAction ensures it keep its underlying action's objects alive
 114 /// as long as the \p IncrementalParser needs them.
 115 ///
 116 class IncrementalAction : public WrapperFrontendAction {
 117 private:
 118   bool IsTerminating = false;
 119
 120 public:
 121   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
 122                     llvm::Error &Err)
 123       : WrapperFrontendAction([&]() {
 124           llvm::ErrorAsOutParameter EAO(&Err);
 125           std::unique_ptr<FrontendAction> Act;
 126           switch (CI.getFrontendOpts().ProgramAction) {
 127           default:
 128             Err = llvm::createStringError(
 129                 std::errc::state_not_recoverable,
 130                 "Driver initialization failed. "
 131                 "Incremental mode for action %d is not supported",
 132                 CI.getFrontendOpts().ProgramAction);
 133             return Act;
 134           case frontend::ASTDump:
 135             [[fallthrough]];
 136           case frontend::ASTPrint:
 137             [[fallthrough]];
 138           case frontend::ParseSyntaxOnly:
 139             Act = CreateFrontendAction(CI);
 140             break;
 141           case frontend::PluginAction:
 142             [[fallthrough]];
 143           case frontend::EmitAssembly:
 144             [[fallthrough]];
 145           case frontend::EmitBC:
 146             [[fallthrough]];
 147           case frontend::EmitObj:
 148             [[fallthrough]];
 149           case frontend::PrintPreprocessedInput:
 150             [[fallthrough]];
 151           case frontend::EmitLLVMOnly:
 152             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
 153             break;
 154           }
 155           return Act;
 156         }()) {}
 157   FrontendAction *getWrapped() const { return WrappedAction.get(); }
 158   TranslationUnitKind getTranslationUnitKind() override {
 159     return TU_Incremental;
 160   }
 161
 162   void ExecuteAction() override {
 163     CompilerInstance &CI = getCompilerInstance();
 164     assert(CI.hasPreprocessor() && "No PP!");
 165
 166     // Use a code completion consumer?
 167     CodeCompleteConsumer *CompletionConsumer = nullptr;
 168     if (CI.hasCodeCompletionConsumer())
 169       CompletionConsumer = &CI.getCodeCompletionConsumer();
 170
 171     Preprocessor &PP = CI.getPreprocessor();
 172     PP.EnterMainSourceFile();
 173
 174     if (!CI.hasSema())
 175       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
 176   }
 177
 178   // Do not terminate after processing the input. This allows us to keep various
 179   // clang objects alive and to incrementally grow the current TU.
 180   void EndSourceFile() override {
 181     // The WrappedAction can be nullptr if we issued an error in the ctor.
 182     if (IsTerminating && getWrapped())
 183       WrapperFrontendAction::EndSourceFile();
 184   }
 185
 186   void FinalizeAction() {
 187     assert(!IsTerminating && "Already finalized!");
 188     IsTerminating = true;
 189     EndSourceFile();
 190   }
 191 };
 192
 193 CodeGenerator *IncrementalParser::getCodeGen() const {
 194   FrontendAction *WrappedAct = Act->getWrapped();
 195   if (!WrappedAct->hasIRSupport())
 196     return nullptr;
 197   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 198 }
 199
 200 IncrementalParser::IncrementalParser() {}
 201
 202 IncrementalParser::IncrementalParser(Interpreter &Interp,
 203                                      std::unique_ptr<CompilerInstance> Instance,
 204                                      llvm::LLVMContext &LLVMCtx,
 205                                      llvm::Error &Err)
 206     : CI(std::move(Instance)) {
 207   llvm::ErrorAsOutParameter EAO(&Err);
 208   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
 209   if (Err)
 210     return;
 211   CI->ExecuteAction(*Act);
 212
 213   if (getCodeGen())
 214     CachedInCodeGenModule = GenModule();
 215
 216   std::unique_ptr<ASTConsumer> IncrConsumer =
 217       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
 218   CI->setASTConsumer(std::move(IncrConsumer));
 219   Consumer = &CI->getASTConsumer();
 220   P.reset(
 221       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
 222   P->Initialize();
 223
 224   // An initial PTU is needed as CUDA includes some headers automatically
 225   auto PTU = ParseOrWrapTopLevelDecl();
 226   if (auto E = PTU.takeError()) {
 227     consumeError(std::move(E)); // FIXME
 228     return;                     // PTU.takeError();
 229   }
 230
 231   if (getCodeGen()) {
 232     PTU->TheModule = GenModule();
 233     assert(PTU->TheModule && "Failed to create initial PTU");
 234   }
 235 }
 236
 237 IncrementalParser::~IncrementalParser() {
 238   P.reset();
 239   Act->FinalizeAction();
 240 }
 241
 242 llvm::Expected<PartialTranslationUnit &>
 243 IncrementalParser::ParseOrWrapTopLevelDecl() {
 244   // Recover resources if we crash before exiting this method.
 245   Sema &S = CI->getSema();
 246   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
 247   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
 248   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
 249
 250   PTUs.emplace_back(PartialTranslationUnit());
 251   PartialTranslationUnit &LastPTU = PTUs.back();
 252   // Add a new PTU.
 253   ASTContext &C = S.getASTContext();
 254   C.addTranslationUnitDecl();
 255   LastPTU.TUPart = C.getTranslationUnitDecl();
 256
 257   // Skip previous eof due to last incremental input.
 258   if (P->getCurToken().is(tok::annot_repl_input_end)) {
 259     P->ConsumeAnyToken();
 260     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
 261     // might want to do that around HandleEndOfTranslationUnit.
 262     P->ExitScope();
 263     S.CurContext = nullptr;
 264     // Start a new PTU.
 265     P->EnterScope(Scope::DeclScope);
 266     S.ActOnTranslationUnitScope(P->getCurScope());
 267   }
 268
 269   Parser::DeclGroupPtrTy ADecl;
 270   Sema::ModuleImportState ImportState;
 271   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
 272        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
 273     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
 274       return llvm::make_error<llvm::StringError>("Parsing failed. "
 275                                                  "The consumer rejected a decl",
 276                                                  std::error_code());
 277   }
 278
 279   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
 280   if (Diags.hasErrorOccurred()) {
 281     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
 282                                             nullptr};
 283     CleanUpPTU(MostRecentPTU);
 284
 285     Diags.Reset(/*soft=*/true);
 286     Diags.getClient()->clear();
 287     return llvm::make_error<llvm::StringError>("Parsing failed.",
 288                                                std::error_code());
 289   }
 290
 291   // Process any TopLevelDecls generated by #pragma weak.
 292   for (Decl *D : S.WeakTopLevelDecls()) {
 293     DeclGroupRef DGR(D);
 294     Consumer->HandleTopLevelDecl(DGR);
 295   }
 296
 297   LocalInstantiations.perform();
 298   GlobalInstantiations.perform();
 299
 300   Consumer->HandleTranslationUnit(C);
 301
 302   return LastPTU;
 303 }
 304
 305 llvm::Expected<PartialTranslationUnit &>
 306 IncrementalParser::Parse(llvm::StringRef input) {
 307   Preprocessor &PP = CI->getPreprocessor();
 308   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 309
 310   std::ostringstream SourceName;
 311   SourceName << "input_line_" << InputCount++;
 312
 313   // Create an uninitialized memory buffer, copy code in and append "\n"
 314   size_t InputSize = input.size(); // don't include trailing 0
 315   // MemBuffer size should *not* include terminating zero
 316   std::unique_ptr<llvm::MemoryBuffer> MB(
 317       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
 318                                                         SourceName.str()));
 319   char *MBStart = const_cast<char *>(MB->getBufferStart());
 320   memcpy(MBStart, input.data(), InputSize);
 321   MBStart[InputSize] = '\n';
 322
 323   SourceManager &SM = CI->getSourceManager();
 324
 325   // FIXME: Create SourceLocation, which will allow clang to order the overload
 326   // candidates for example
 327   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
 328
 329   // Create FileID for the current buffer.
 330   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
 331                                /*LoadedOffset=*/0, NewLoc);
 332
 333   // NewLoc only used for diags.
 334   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
 335     return llvm::make_error<llvm::StringError>("Parsing failed. "
 336                                                "Cannot enter source file.",
 337                                                std::error_code());
 338
 339   auto PTU = ParseOrWrapTopLevelDecl();
 340   if (!PTU)
 341     return PTU.takeError();
 342
 343   if (PP.getLangOpts().DelayedTemplateParsing) {
 344     // Microsoft-specific:
 345     // Late parsed templates can leave unswallowed "macro"-like tokens.
 346     // They will seriously confuse the Parser when entering the next
 347     // source file. So lex until we are EOF.
 348     Token Tok;
 349     do {
 350       PP.Lex(Tok);
 351     } while (Tok.isNot(tok::annot_repl_input_end));
 352   } else {
 353     Token AssertTok;
 354     PP.Lex(AssertTok);
 355     assert(AssertTok.is(tok::annot_repl_input_end) &&
 356            "Lexer must be EOF when starting incremental parse!");
 357   }
 358
 359   if (std::unique_ptr<llvm::Module> M = GenModule())
 360     PTU->TheModule = std::move(M);
 361
 362   return PTU;
 363 }
 364
 365 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
 366   static unsigned ID = 0;
 367   if (CodeGenerator *CG = getCodeGen()) {
 368     // Clang's CodeGen is designed to work with a single llvm::Module. In many
 369     // cases for convenience various CodeGen parts have a reference to the
 370     // llvm::Module (TheModule or Module) which does not change when a new
 371     // module is pushed. However, the execution engine wants to take ownership
 372     // of the module which does not map well to CodeGen's design. To work this
 373     // around we created an empty module to make CodeGen happy. We should make
 374     // sure it always stays empty.
 375     assert((!CachedInCodeGenModule ||
 376             (CachedInCodeGenModule->empty() &&
 377              CachedInCodeGenModule->global_empty() &&
 378              CachedInCodeGenModule->alias_empty() &&
 379              CachedInCodeGenModule->ifunc_empty())) &&
 380            "CodeGen wrote to a readonly module");
 381     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
 382     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
 383     return M;
 384   }
 385   return nullptr;
 386 }
 387
 388 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
 389   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
 390   if (StoredDeclsMap *Map = MostRecentTU->getPrimaryContext()->getLookupPtr()) {
 391     for (auto &&[Key, List] : *Map) {
 392       DeclContextLookupResult R = List.getLookupResult();
 393       std::vector<NamedDecl *> NamedDeclsToRemove;
 394       bool RemoveAll = true;
 395       for (NamedDecl *D : R) {
 396         if (D->getTranslationUnitDecl() == MostRecentTU)
 397           NamedDeclsToRemove.push_back(D);
 398         else
 399           RemoveAll = false;
 400       }
 401       if (LLVM_LIKELY(RemoveAll)) {
 402         Map->erase(Key);
 403       } else {
 404         for (NamedDecl *D : NamedDeclsToRemove)
 405           List.remove(D);
 406       }
 407     }
 408   }
 409
 410   // FIXME: We should de-allocate MostRecentTU
 411   for (Decl *D : MostRecentTU->decls()) {
 412     auto *ND = dyn_cast<NamedDecl>(D);
 413     if (!ND)
 414       continue;
 415     // Check if we need to clean up the IdResolver chain.
 416     if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
 417         !D->getLangOpts().CPlusPlus)
 418       getCI()->getSema().IdResolver.RemoveDecl(ND);
 419   }
 420 }
 421
 422 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
 423   CodeGenerator *CG = getCodeGen();
 424   assert(CG);
 425   return CG->GetMangledName(GD);
 426 }
 427 } // end namespace clang