clang/lib/Interpreter/IncrementalParser.cpp

   1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the class which performs incremental code compilation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "IncrementalParser.h"
  14
  15 #include "clang/AST/DeclContextInternals.h"
  16 #include "clang/CodeGen/BackendUtil.h"
  17 #include "clang/CodeGen/CodeGenAction.h"
  18 #include "clang/CodeGen/ModuleBuilder.h"
  19 #include "clang/Frontend/CompilerInstance.h"
  20 #include "clang/Frontend/FrontendAction.h"
  21 #include "clang/FrontendTool/Utils.h"
  22 #include "clang/Interpreter/Interpreter.h"
  23 #include "clang/Parse/Parser.h"
  24 #include "clang/Sema/Sema.h"
  25 #include "llvm/Option/ArgList.h"
  26 #include "llvm/Support/CrashRecoveryContext.h"
  27 #include "llvm/Support/Error.h"
  28 #include "llvm/Support/Timer.h"
  29
  30 #include <sstream>
  31
  32 namespace clang {
  33
  34 class IncrementalASTConsumer final : public ASTConsumer {
  35   Interpreter &Interp;
  36   std::unique_ptr<ASTConsumer> Consumer;
  37
  38 public:
  39   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
  40       : Interp(InterpRef), Consumer(std::move(C)) {}
  41
  42   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
  43     if (DGR.isNull())
  44       return true;
  45     if (!Consumer)
  46       return true;
  47
  48     for (Decl *D : DGR)
  49       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
  50           TSD && TSD->isSemiMissing())
  51         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
  52
  53     return Consumer->HandleTopLevelDecl(DGR);
  54   }
  55   void HandleTranslationUnit(ASTContext &Ctx) override final {
  56     Consumer->HandleTranslationUnit(Ctx);
  57   }
  58   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
  59     Consumer->HandleInlineFunctionDefinition(D);
  60   }
  61   void HandleInterestingDecl(DeclGroupRef D) override final {
  62     Consumer->HandleInterestingDecl(D);
  63   }
  64   void HandleTagDeclDefinition(TagDecl *D) override final {
  65     Consumer->HandleTagDeclDefinition(D);
  66   }
  67   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
  68     Consumer->HandleTagDeclRequiredDefinition(D);
  69   }
  70   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
  71     Consumer->HandleCXXImplicitFunctionInstantiation(D);
  72   }
  73   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
  74     Consumer->HandleTopLevelDeclInObjCContainer(D);
  75   }
  76   void HandleImplicitImportDecl(ImportDecl *D) override final {
  77     Consumer->HandleImplicitImportDecl(D);
  78   }
  79   void CompleteTentativeDefinition(VarDecl *D) override final {
  80     Consumer->CompleteTentativeDefinition(D);
  81   }
  82   void CompleteExternalDeclaration(VarDecl *D) override final {
  83     Consumer->CompleteExternalDeclaration(D);
  84   }
  85   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
  86     Consumer->AssignInheritanceModel(RD);
  87   }
  88   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
  89     Consumer->HandleCXXStaticMemberVarInstantiation(D);
  90   }
  91   void HandleVTable(CXXRecordDecl *RD) override final {
  92     Consumer->HandleVTable(RD);
  93   }
  94   ASTMutationListener *GetASTMutationListener() override final {
  95     return Consumer->GetASTMutationListener();
  96   }
  97   ASTDeserializationListener *GetASTDeserializationListener() override final {
  98     return Consumer->GetASTDeserializationListener();
  99   }
 100   void PrintStats() override final { Consumer->PrintStats(); }
 101   bool shouldSkipFunctionBody(Decl *D) override final {
 102     return Consumer->shouldSkipFunctionBody(D);
 103   }
 104   static bool classof(const clang::ASTConsumer *) { return true; }
 105 };
 106
 107 /// A custom action enabling the incremental processing functionality.
 108 ///
 109 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
 110 /// sees a call to \p EndSourceFile it deletes some of the important objects
 111 /// such as \p Preprocessor and \p Sema assuming no further input will come.
 112 ///
 113 /// \p IncrementalAction ensures it keep its underlying action's objects alive
 114 /// as long as the \p IncrementalParser needs them.
 115 ///
 116 class IncrementalAction : public WrapperFrontendAction {
 117 private:
 118   bool IsTerminating = false;
 119
 120 public:
 121   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
 122                     llvm::Error &Err)
 123       : WrapperFrontendAction([&]() {
 124           llvm::ErrorAsOutParameter EAO(&Err);
 125           std::unique_ptr<FrontendAction> Act;
 126           switch (CI.getFrontendOpts().ProgramAction) {
 127           default:
 128             Err = llvm::createStringError(
 129                 std::errc::state_not_recoverable,
 130                 "Driver initialization failed. "
 131                 "Incremental mode for action %d is not supported",
 132                 CI.getFrontendOpts().ProgramAction);
 133             return Act;
 134           case frontend::ASTDump:
 135             [[fallthrough]];
 136           case frontend::ASTPrint:
 137             [[fallthrough]];
 138           case frontend::ParseSyntaxOnly:
 139             Act = CreateFrontendAction(CI);
 140             break;
 141           case frontend::PluginAction:
 142             [[fallthrough]];
 143           case frontend::EmitAssembly:
 144             [[fallthrough]];
 145           case frontend::EmitBC:
 146             [[fallthrough]];
 147           case frontend::EmitObj:
 148             [[fallthrough]];
 149           case frontend::PrintPreprocessedInput:
 150             [[fallthrough]];
 151           case frontend::EmitLLVMOnly:
 152             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
 153             break;
 154           }
 155           return Act;
 156         }()) {}
 157   FrontendAction *getWrapped() const { return WrappedAction.get(); }
 158   TranslationUnitKind getTranslationUnitKind() override {
 159     return TU_Incremental;
 160   }
 161
 162   void ExecuteAction() override {
 163     CompilerInstance &CI = getCompilerInstance();
 164     assert(CI.hasPreprocessor() && "No PP!");
 165
 166     // Use a code completion consumer?
 167     CodeCompleteConsumer *CompletionConsumer = nullptr;
 168     if (CI.hasCodeCompletionConsumer())
 169       CompletionConsumer = &CI.getCodeCompletionConsumer();
 170
 171     Preprocessor &PP = CI.getPreprocessor();
 172     PP.EnterMainSourceFile();
 173
 174     if (!CI.hasSema())
 175       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
 176   }
 177
 178   // Do not terminate after processing the input. This allows us to keep various
 179   // clang objects alive and to incrementally grow the current TU.
 180   void EndSourceFile() override {
 181     // The WrappedAction can be nullptr if we issued an error in the ctor.
 182     if (IsTerminating && getWrapped())
 183       WrapperFrontendAction::EndSourceFile();
 184   }
 185
 186   void FinalizeAction() {
 187     assert(!IsTerminating && "Already finalized!");
 188     IsTerminating = true;
 189     EndSourceFile();
 190   }
 191 };
 192
 193 CodeGenerator *IncrementalParser::getCodeGen() const {
 194   FrontendAction *WrappedAct = Act->getWrapped();
 195   if (!WrappedAct->hasIRSupport())
 196     return nullptr;
 197   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 198 }
 199
 200 IncrementalParser::IncrementalParser() {}
 201
 202 IncrementalParser::IncrementalParser(Interpreter &Interp,
 203                                      std::unique_ptr<CompilerInstance> Instance,
 204                                      llvm::LLVMContext &LLVMCtx,
 205                                      llvm::Error &Err)
 206     : CI(std::move(Instance)) {
 207   llvm::ErrorAsOutParameter EAO(&Err);
 208   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
 209   if (Err)
 210     return;
 211   CI->ExecuteAction(*Act);
 212   std::unique_ptr<ASTConsumer> IncrConsumer =
 213       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
 214   CI->setASTConsumer(std::move(IncrConsumer));
 215   Consumer = &CI->getASTConsumer();
 216   P.reset(
 217       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
 218   P->Initialize();
 219
 220   // An initial PTU is needed as CUDA includes some headers automatically
 221   auto PTU = ParseOrWrapTopLevelDecl();
 222   if (auto E = PTU.takeError()) {
 223     consumeError(std::move(E)); // FIXME
 224     return;                     // PTU.takeError();
 225   }
 226
 227   if (CodeGenerator *CG = getCodeGen()) {
 228     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
 229     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
 230                     M->getContext());
 231     PTU->TheModule = std::move(M);
 232     assert(PTU->TheModule && "Failed to create initial PTU");
 233   }
 234 }
 235
 236 IncrementalParser::~IncrementalParser() {
 237   P.reset();
 238   Act->FinalizeAction();
 239 }
 240
 241 llvm::Expected<PartialTranslationUnit &>
 242 IncrementalParser::ParseOrWrapTopLevelDecl() {
 243   // Recover resources if we crash before exiting this method.
 244   Sema &S = CI->getSema();
 245   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
 246   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
 247   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
 248
 249   PTUs.emplace_back(PartialTranslationUnit());
 250   PartialTranslationUnit &LastPTU = PTUs.back();
 251   // Add a new PTU.
 252   ASTContext &C = S.getASTContext();
 253   C.addTranslationUnitDecl();
 254   LastPTU.TUPart = C.getTranslationUnitDecl();
 255
 256   // Skip previous eof due to last incremental input.
 257   if (P->getCurToken().is(tok::annot_repl_input_end)) {
 258     P->ConsumeAnyToken();
 259     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
 260     // might want to do that around HandleEndOfTranslationUnit.
 261     P->ExitScope();
 262     S.CurContext = nullptr;
 263     // Start a new PTU.
 264     P->EnterScope(Scope::DeclScope);
 265     S.ActOnTranslationUnitScope(P->getCurScope());
 266   }
 267
 268   Parser::DeclGroupPtrTy ADecl;
 269   Sema::ModuleImportState ImportState;
 270   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
 271        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
 272     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
 273       return llvm::make_error<llvm::StringError>("Parsing failed. "
 274                                                  "The consumer rejected a decl",
 275                                                  std::error_code());
 276   }
 277
 278   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
 279   if (Diags.hasErrorOccurred()) {
 280     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
 281                                             nullptr};
 282     CleanUpPTU(MostRecentPTU);
 283
 284     Diags.Reset(/*soft=*/true);
 285     Diags.getClient()->clear();
 286     return llvm::make_error<llvm::StringError>("Parsing failed.",
 287                                                std::error_code());
 288   }
 289
 290   // Process any TopLevelDecls generated by #pragma weak.
 291   for (Decl *D : S.WeakTopLevelDecls()) {
 292     DeclGroupRef DGR(D);
 293     Consumer->HandleTopLevelDecl(DGR);
 294   }
 295
 296   LocalInstantiations.perform();
 297   GlobalInstantiations.perform();
 298
 299   Consumer->HandleTranslationUnit(C);
 300
 301   return LastPTU;
 302 }
 303
 304 llvm::Expected<PartialTranslationUnit &>
 305 IncrementalParser::Parse(llvm::StringRef input) {
 306   Preprocessor &PP = CI->getPreprocessor();
 307   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 308
 309   std::ostringstream SourceName;
 310   SourceName << "input_line_" << InputCount++;
 311
 312   // Create an uninitialized memory buffer, copy code in and append "\n"
 313   size_t InputSize = input.size(); // don't include trailing 0
 314   // MemBuffer size should *not* include terminating zero
 315   std::unique_ptr<llvm::MemoryBuffer> MB(
 316       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
 317                                                         SourceName.str()));
 318   char *MBStart = const_cast<char *>(MB->getBufferStart());
 319   memcpy(MBStart, input.data(), InputSize);
 320   MBStart[InputSize] = '\n';
 321
 322   SourceManager &SM = CI->getSourceManager();
 323
 324   // FIXME: Create SourceLocation, which will allow clang to order the overload
 325   // candidates for example
 326   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
 327
 328   // Create FileID for the current buffer.
 329   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
 330                                /*LoadedOffset=*/0, NewLoc);
 331
 332   // NewLoc only used for diags.
 333   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
 334     return llvm::make_error<llvm::StringError>("Parsing failed. "
 335                                                "Cannot enter source file.",
 336                                                std::error_code());
 337
 338   auto PTU = ParseOrWrapTopLevelDecl();
 339   if (!PTU)
 340     return PTU.takeError();
 341
 342   if (PP.getLangOpts().DelayedTemplateParsing) {
 343     // Microsoft-specific:
 344     // Late parsed templates can leave unswallowed "macro"-like tokens.
 345     // They will seriously confuse the Parser when entering the next
 346     // source file. So lex until we are EOF.
 347     Token Tok;
 348     do {
 349       PP.Lex(Tok);
 350     } while (Tok.isNot(tok::annot_repl_input_end));
 351   } else {
 352     Token AssertTok;
 353     PP.Lex(AssertTok);
 354     assert(AssertTok.is(tok::annot_repl_input_end) &&
 355            "Lexer must be EOF when starting incremental parse!");
 356   }
 357
 358   if (std::unique_ptr<llvm::Module> M = GenModule())
 359     PTU->TheModule = std::move(M);
 360
 361   return PTU;
 362 }
 363
 364 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
 365   static unsigned ID = 0;
 366   if (CodeGenerator *CG = getCodeGen()) {
 367     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
 368     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
 369     return M;
 370   }
 371   return nullptr;
 372 }
 373
 374 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
 375   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
 376   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
 377   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
 378     for (auto I = Map->begin(); I != Map->end(); ++I) {
 379       StoredDeclsList &List = I->second;
 380       DeclContextLookupResult R = List.getLookupResult();
 381       for (NamedDecl *D : R) {
 382         if (D->getTranslationUnitDecl() == MostRecentTU) {
 383           List.remove(D);
 384         }
 385       }
 386       if (List.isNull())
 387         Map->erase(I);
 388     }
 389   }
 390 }
 391
 392 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
 393   CodeGenerator *CG = getCodeGen();
 394   assert(CG);
 395   return CG->GetMangledName(GD);
 396 }
 397 } // end namespace clang