clang/lib/Interpreter/IncrementalParser.cpp

   1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the class which performs incremental code compilation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "IncrementalParser.h"
  14 #include "clang/AST/DeclContextInternals.h"
  15 #include "clang/CodeGen/BackendUtil.h"
  16 #include "clang/CodeGen/CodeGenAction.h"
  17 #include "clang/CodeGen/ModuleBuilder.h"
  18 #include "clang/Frontend/CompilerInstance.h"
  19 #include "clang/Frontend/FrontendAction.h"
  20 #include "clang/FrontendTool/Utils.h"
  21 #include "clang/Interpreter/Interpreter.h"
  22 #include "clang/Parse/Parser.h"
  23 #include "clang/Sema/Sema.h"
  24 #include "llvm/Option/ArgList.h"
  25 #include "llvm/Support/CrashRecoveryContext.h"
  26 #include "llvm/Support/Error.h"
  27 #include "llvm/Support/Timer.h"
  28
  29 #include <sstream>
  30
  31 namespace clang {
  32
  33 class IncrementalASTConsumer final : public ASTConsumer {
  34   Interpreter &Interp;
  35   std::unique_ptr<ASTConsumer> Consumer;
  36
  37 public:
  38   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
  39       : Interp(InterpRef), Consumer(std::move(C)) {}
  40
  41   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
  42     if (DGR.isNull())
  43       return true;
  44     if (!Consumer)
  45       return true;
  46
  47     for (Decl *D : DGR)
  48       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
  49           TSD && TSD->isSemiMissing())
  50         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
  51
  52     return Consumer->HandleTopLevelDecl(DGR);
  53   }
  54   void HandleTranslationUnit(ASTContext &Ctx) override final {
  55     Consumer->HandleTranslationUnit(Ctx);
  56   }
  57   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
  58     Consumer->HandleInlineFunctionDefinition(D);
  59   }
  60   void HandleInterestingDecl(DeclGroupRef D) override final {
  61     Consumer->HandleInterestingDecl(D);
  62   }
  63   void HandleTagDeclDefinition(TagDecl *D) override final {
  64     Consumer->HandleTagDeclDefinition(D);
  65   }
  66   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
  67     Consumer->HandleTagDeclRequiredDefinition(D);
  68   }
  69   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
  70     Consumer->HandleCXXImplicitFunctionInstantiation(D);
  71   }
  72   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
  73     Consumer->HandleTopLevelDeclInObjCContainer(D);
  74   }
  75   void HandleImplicitImportDecl(ImportDecl *D) override final {
  76     Consumer->HandleImplicitImportDecl(D);
  77   }
  78   void CompleteTentativeDefinition(VarDecl *D) override final {
  79     Consumer->CompleteTentativeDefinition(D);
  80   }
  81   void CompleteExternalDeclaration(VarDecl *D) override final {
  82     Consumer->CompleteExternalDeclaration(D);
  83   }
  84   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
  85     Consumer->AssignInheritanceModel(RD);
  86   }
  87   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
  88     Consumer->HandleCXXStaticMemberVarInstantiation(D);
  89   }
  90   void HandleVTable(CXXRecordDecl *RD) override final {
  91     Consumer->HandleVTable(RD);
  92   }
  93   ASTMutationListener *GetASTMutationListener() override final {
  94     return Consumer->GetASTMutationListener();
  95   }
  96   ASTDeserializationListener *GetASTDeserializationListener() override final {
  97     return Consumer->GetASTDeserializationListener();
  98   }
  99   void PrintStats() override final { Consumer->PrintStats(); }
 100   bool shouldSkipFunctionBody(Decl *D) override final {
 101     return Consumer->shouldSkipFunctionBody(D);
 102   }
 103   static bool classof(const clang::ASTConsumer *) { return true; }
 104 };
 105
 106 /// A custom action enabling the incremental processing functionality.
 107 ///
 108 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
 109 /// sees a call to \p EndSourceFile it deletes some of the important objects
 110 /// such as \p Preprocessor and \p Sema assuming no further input will come.
 111 ///
 112 /// \p IncrementalAction ensures it keep its underlying action's objects alive
 113 /// as long as the \p IncrementalParser needs them.
 114 ///
 115 class IncrementalAction : public WrapperFrontendAction {
 116 private:
 117   bool IsTerminating = false;
 118
 119 public:
 120   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
 121                     llvm::Error &Err)
 122       : WrapperFrontendAction([&]() {
 123           llvm::ErrorAsOutParameter EAO(&Err);
 124           std::unique_ptr<FrontendAction> Act;
 125           switch (CI.getFrontendOpts().ProgramAction) {
 126           default:
 127             Err = llvm::createStringError(
 128                 std::errc::state_not_recoverable,
 129                 "Driver initialization failed. "
 130                 "Incremental mode for action %d is not supported",
 131                 CI.getFrontendOpts().ProgramAction);
 132             return Act;
 133           case frontend::ASTDump:
 134             [[fallthrough]];
 135           case frontend::ASTPrint:
 136             [[fallthrough]];
 137           case frontend::ParseSyntaxOnly:
 138             Act = CreateFrontendAction(CI);
 139             break;
 140           case frontend::PluginAction:
 141             [[fallthrough]];
 142           case frontend::EmitAssembly:
 143             [[fallthrough]];
 144           case frontend::EmitBC:
 145             [[fallthrough]];
 146           case frontend::EmitObj:
 147             [[fallthrough]];
 148           case frontend::PrintPreprocessedInput:
 149             [[fallthrough]];
 150           case frontend::EmitLLVMOnly:
 151             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
 152             break;
 153           }
 154           return Act;
 155         }()) {}
 156   FrontendAction *getWrapped() const { return WrappedAction.get(); }
 157   TranslationUnitKind getTranslationUnitKind() override {
 158     return TU_Incremental;
 159   }
 160   void ExecuteAction() override {
 161     CompilerInstance &CI = getCompilerInstance();
 162     assert(CI.hasPreprocessor() && "No PP!");
 163
 164     // FIXME: Move the truncation aspect of this into Sema, we delayed this till
 165     // here so the source manager would be initialized.
 166     if (hasCodeCompletionSupport() &&
 167         !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
 168       CI.createCodeCompletionConsumer();
 169
 170     // Use a code completion consumer?
 171     CodeCompleteConsumer *CompletionConsumer = nullptr;
 172     if (CI.hasCodeCompletionConsumer())
 173       CompletionConsumer = &CI.getCodeCompletionConsumer();
 174
 175     Preprocessor &PP = CI.getPreprocessor();
 176     PP.EnterMainSourceFile();
 177
 178     if (!CI.hasSema())
 179       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
 180   }
 181
 182   // Do not terminate after processing the input. This allows us to keep various
 183   // clang objects alive and to incrementally grow the current TU.
 184   void EndSourceFile() override {
 185     // The WrappedAction can be nullptr if we issued an error in the ctor.
 186     if (IsTerminating && getWrapped())
 187       WrapperFrontendAction::EndSourceFile();
 188   }
 189
 190   void FinalizeAction() {
 191     assert(!IsTerminating && "Already finalized!");
 192     IsTerminating = true;
 193     EndSourceFile();
 194   }
 195 };
 196
 197 CodeGenerator *IncrementalParser::getCodeGen() const {
 198   FrontendAction *WrappedAct = Act->getWrapped();
 199   if (!WrappedAct->hasIRSupport())
 200     return nullptr;
 201   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 202 }
 203
 204 IncrementalParser::IncrementalParser() {}
 205
 206 IncrementalParser::IncrementalParser(Interpreter &Interp,
 207                                      std::unique_ptr<CompilerInstance> Instance,
 208                                      llvm::LLVMContext &LLVMCtx,
 209                                      llvm::Error &Err)
 210     : CI(std::move(Instance)) {
 211   llvm::ErrorAsOutParameter EAO(&Err);
 212   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
 213   if (Err)
 214     return;
 215   CI->ExecuteAction(*Act);
 216   std::unique_ptr<ASTConsumer> IncrConsumer =
 217       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
 218   CI->setASTConsumer(std::move(IncrConsumer));
 219   Consumer = &CI->getASTConsumer();
 220   P.reset(
 221       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
 222   P->Initialize();
 223
 224   // An initial PTU is needed as CUDA includes some headers automatically
 225   auto PTU = ParseOrWrapTopLevelDecl();
 226   if (auto E = PTU.takeError()) {
 227     consumeError(std::move(E)); // FIXME
 228     return;                     // PTU.takeError();
 229   }
 230
 231   if (CodeGenerator *CG = getCodeGen()) {
 232     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
 233     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
 234                     M->getContext());
 235     PTU->TheModule = std::move(M);
 236     assert(PTU->TheModule && "Failed to create initial PTU");
 237   }
 238 }
 239
 240 IncrementalParser::~IncrementalParser() {
 241   P.reset();
 242   Act->FinalizeAction();
 243 }
 244
 245 llvm::Expected<PartialTranslationUnit &>
 246 IncrementalParser::ParseOrWrapTopLevelDecl() {
 247   // Recover resources if we crash before exiting this method.
 248   Sema &S = CI->getSema();
 249   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
 250   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
 251   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
 252
 253   PTUs.emplace_back(PartialTranslationUnit());
 254   PartialTranslationUnit &LastPTU = PTUs.back();
 255   // Add a new PTU.
 256   ASTContext &C = S.getASTContext();
 257   C.addTranslationUnitDecl();
 258   LastPTU.TUPart = C.getTranslationUnitDecl();
 259
 260   // Skip previous eof due to last incremental input.
 261   if (P->getCurToken().is(tok::annot_repl_input_end)) {
 262     P->ConsumeAnyToken();
 263     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
 264     // might want to do that around HandleEndOfTranslationUnit.
 265     P->ExitScope();
 266     S.CurContext = nullptr;
 267     // Start a new PTU.
 268     P->EnterScope(Scope::DeclScope);
 269     S.ActOnTranslationUnitScope(P->getCurScope());
 270   }
 271
 272   Parser::DeclGroupPtrTy ADecl;
 273   Sema::ModuleImportState ImportState;
 274   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
 275        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
 276     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
 277       return llvm::make_error<llvm::StringError>("Parsing failed. "
 278                                                  "The consumer rejected a decl",
 279                                                  std::error_code());
 280   }
 281
 282   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
 283   if (Diags.hasErrorOccurred()) {
 284     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
 285                                             nullptr};
 286     CleanUpPTU(MostRecentPTU);
 287
 288     Diags.Reset(/*soft=*/true);
 289     Diags.getClient()->clear();
 290     return llvm::make_error<llvm::StringError>("Parsing failed.",
 291                                                std::error_code());
 292   }
 293
 294   // Process any TopLevelDecls generated by #pragma weak.
 295   for (Decl *D : S.WeakTopLevelDecls()) {
 296     DeclGroupRef DGR(D);
 297     Consumer->HandleTopLevelDecl(DGR);
 298   }
 299
 300   LocalInstantiations.perform();
 301   GlobalInstantiations.perform();
 302
 303   Consumer->HandleTranslationUnit(C);
 304
 305   return LastPTU;
 306 }
 307
 308 llvm::Expected<PartialTranslationUnit &>
 309 IncrementalParser::Parse(llvm::StringRef input) {
 310   Preprocessor &PP = CI->getPreprocessor();
 311   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 312
 313   std::ostringstream SourceName;
 314   SourceName << "input_line_" << InputCount++;
 315
 316   // Create an uninitialized memory buffer, copy code in and append "\n"
 317   size_t InputSize = input.size(); // don't include trailing 0
 318   // MemBuffer size should *not* include terminating zero
 319   std::unique_ptr<llvm::MemoryBuffer> MB(
 320       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
 321                                                         SourceName.str()));
 322   char *MBStart = const_cast<char *>(MB->getBufferStart());
 323   memcpy(MBStart, input.data(), InputSize);
 324   MBStart[InputSize] = '\n';
 325
 326   SourceManager &SM = CI->getSourceManager();
 327
 328   // FIXME: Create SourceLocation, which will allow clang to order the overload
 329   // candidates for example
 330   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
 331
 332   // Create FileID for the current buffer.
 333   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
 334                                /*LoadedOffset=*/0, NewLoc);
 335
 336   // NewLoc only used for diags.
 337   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
 338     return llvm::make_error<llvm::StringError>("Parsing failed. "
 339                                                "Cannot enter source file.",
 340                                                std::error_code());
 341
 342   auto PTU = ParseOrWrapTopLevelDecl();
 343   if (!PTU)
 344     return PTU.takeError();
 345
 346   if (PP.getLangOpts().DelayedTemplateParsing) {
 347     // Microsoft-specific:
 348     // Late parsed templates can leave unswallowed "macro"-like tokens.
 349     // They will seriously confuse the Parser when entering the next
 350     // source file. So lex until we are EOF.
 351     Token Tok;
 352     do {
 353       PP.Lex(Tok);
 354     } while (Tok.isNot(tok::annot_repl_input_end));
 355   } else {
 356     Token AssertTok;
 357     PP.Lex(AssertTok);
 358     assert(AssertTok.is(tok::annot_repl_input_end) &&
 359            "Lexer must be EOF when starting incremental parse!");
 360   }
 361
 362   if (std::unique_ptr<llvm::Module> M = GenModule())
 363     PTU->TheModule = std::move(M);
 364
 365   return PTU;
 366 }
 367
 368 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
 369   static unsigned ID = 0;
 370   if (CodeGenerator *CG = getCodeGen()) {
 371     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
 372     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
 373     return M;
 374   }
 375   return nullptr;
 376 }
 377
 378 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
 379   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
 380   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
 381   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
 382     for (auto I = Map->begin(); I != Map->end(); ++I) {
 383       StoredDeclsList &List = I->second;
 384       DeclContextLookupResult R = List.getLookupResult();
 385       for (NamedDecl *D : R) {
 386         if (D->getTranslationUnitDecl() == MostRecentTU) {
 387           List.remove(D);
 388         }
 389       }
 390       if (List.isNull())
 391         Map->erase(I);
 392     }
 393   }
 394 }
 395
 396 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
 397   CodeGenerator *CG = getCodeGen();
 398   assert(CG);
 399   return CG->GetMangledName(GD);
 400 }
 401
 402 } // end namespace clang