clang/lib/CrossTU/CrossTranslationUnit.cpp

   1 //===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 //  This file implements the CrossTranslationUnit interface.
  10 //
  11 //===----------------------------------------------------------------------===//
  12 #include "clang/CrossTU/CrossTranslationUnit.h"
  13 #include "clang/AST/ASTImporter.h"
  14 #include "clang/AST/Decl.h"
  15 #include "clang/AST/ParentMapContext.h"
  16 #include "clang/Basic/TargetInfo.h"
  17 #include "clang/CrossTU/CrossTUDiagnostic.h"
  18 #include "clang/Frontend/ASTUnit.h"
  19 #include "clang/Frontend/CompilerInstance.h"
  20 #include "clang/Frontend/TextDiagnosticPrinter.h"
  21 #include "clang/Index/USRGeneration.h"
  22 #include "llvm/ADT/Optional.h"
  23 #include "llvm/ADT/Statistic.h"
  24 #include "llvm/ADT/Triple.h"
  25 #include "llvm/Option/ArgList.h"
  26 #include "llvm/Support/ErrorHandling.h"
  27 #include "llvm/Support/ManagedStatic.h"
  28 #include "llvm/Support/Path.h"
  29 #include "llvm/Support/YAMLParser.h"
  30 #include "llvm/Support/raw_ostream.h"
  31 #include <algorithm>
  32 #include <fstream>
  33 #include <sstream>
  34 #include <tuple>
  35
  36 namespace clang {
  37 namespace cross_tu {
  38
  39 namespace {
  40
  41 #define DEBUG_TYPE "CrossTranslationUnit"
  42 STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
  43 STATISTIC(
  44     NumNotInOtherTU,
  45     "The # of getCTUDefinition called but the function is not in any other TU");
  46 STATISTIC(NumGetCTUSuccess,
  47           "The # of getCTUDefinition successfully returned the "
  48           "requested function's body");
  49 STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
  50                                    "encountered an unsupported AST Node");
  51 STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
  52                             "encountered an ODR error");
  53 STATISTIC(NumTripleMismatch, "The # of triple mismatches");
  54 STATISTIC(NumLangMismatch, "The # of language mismatches");
  55 STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
  56 STATISTIC(NumASTLoadThresholdReached,
  57           "The # of ASTs not loaded because of threshold");
  58
  59 // Same as Triple's equality operator, but we check a field only if that is
  60 // known in both instances.
  61 bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
  62   using llvm::Triple;
  63   if (Lhs.getArch() != Triple::UnknownArch &&
  64       Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
  65     return false;
  66   if (Lhs.getSubArch() != Triple::NoSubArch &&
  67       Rhs.getSubArch() != Triple::NoSubArch &&
  68       Lhs.getSubArch() != Rhs.getSubArch())
  69     return false;
  70   if (Lhs.getVendor() != Triple::UnknownVendor &&
  71       Rhs.getVendor() != Triple::UnknownVendor &&
  72       Lhs.getVendor() != Rhs.getVendor())
  73     return false;
  74   if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
  75       Lhs.getOS() != Rhs.getOS())
  76     return false;
  77   if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
  78       Rhs.getEnvironment() != Triple::UnknownEnvironment &&
  79       Lhs.getEnvironment() != Rhs.getEnvironment())
  80     return false;
  81   if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
  82       Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
  83       Lhs.getObjectFormat() != Rhs.getObjectFormat())
  84     return false;
  85   return true;
  86 }
  87
  88 // FIXME: This class is will be removed after the transition to llvm::Error.
  89 class IndexErrorCategory : public std::error_category {
  90 public:
  91   const char *name() const noexcept override { return "clang.index"; }
  92
  93   std::string message(int Condition) const override {
  94     switch (static_cast<index_error_code>(Condition)) {
  95     case index_error_code::success:
  96       // There should not be a success error. Jump to unreachable directly.
  97       // Add this case to make the compiler stop complaining.
  98       break;
  99     case index_error_code::unspecified:
 100       return "An unknown error has occurred.";
 101     case index_error_code::missing_index_file:
 102       return "The index file is missing.";
 103     case index_error_code::invalid_index_format:
 104       return "Invalid index file format.";
 105     case index_error_code::multiple_definitions:
 106       return "Multiple definitions in the index file.";
 107     case index_error_code::missing_definition:
 108       return "Missing definition from the index file.";
 109     case index_error_code::failed_import:
 110       return "Failed to import the definition.";
 111     case index_error_code::failed_to_get_external_ast:
 112       return "Failed to load external AST source.";
 113     case index_error_code::failed_to_generate_usr:
 114       return "Failed to generate USR.";
 115     case index_error_code::triple_mismatch:
 116       return "Triple mismatch";
 117     case index_error_code::lang_mismatch:
 118       return "Language mismatch";
 119     case index_error_code::lang_dialect_mismatch:
 120       return "Language dialect mismatch";
 121     case index_error_code::load_threshold_reached:
 122       return "Load threshold reached";
 123     case index_error_code::invocation_list_ambiguous:
 124       return "Invocation list file contains multiple references to the same "
 125              "source file.";
 126     case index_error_code::invocation_list_file_not_found:
 127       return "Invocation list file is not found.";
 128     case index_error_code::invocation_list_empty:
 129       return "Invocation list file is empty.";
 130     case index_error_code::invocation_list_wrong_format:
 131       return "Invocation list file is in wrong format.";
 132     case index_error_code::invocation_list_lookup_unsuccessful:
 133       return "Invocation list file does not contain the requested source file.";
 134     }
 135     llvm_unreachable("Unrecognized index_error_code.");
 136   }
 137 };
 138
 139 static llvm::ManagedStatic<IndexErrorCategory> Category;
 140 } // end anonymous namespace
 141
 142 char IndexError::ID;
 143
 144 void IndexError::log(raw_ostream &OS) const {
 145   OS << Category->message(static_cast<int>(Code)) << '\n';
 146 }
 147
 148 std::error_code IndexError::convertToErrorCode() const {
 149   return std::error_code(static_cast<int>(Code), *Category);
 150 }
 151
 152 /// Parse one line of the input CTU index file.
 153 ///
 154 /// @param[in]  LineRef     The input CTU index item in format
 155 ///                         "<USR-Length>:<USR> <File-Path>".
 156 /// @param[out] LookupName  The lookup name in format "<USR-Length>:<USR>".
 157 /// @param[out] FilePath    The file path "<File-Path>".
 158 static bool parseCrossTUIndexItem(StringRef LineRef, StringRef &LookupName,
 159                                   StringRef &FilePath) {
 160   // `LineRef` is "<USR-Length>:<USR> <File-Path>" now.
 161
 162   size_t USRLength = 0;
 163   if (LineRef.consumeInteger(10, USRLength))
 164     return false;
 165   assert(USRLength && "USRLength should be greater than zero.");
 166
 167   if (!LineRef.consume_front(":"))
 168     return false;
 169
 170   // `LineRef` is now just "<USR> <File-Path>".
 171
 172   // Check LookupName length out of bound and incorrect delimiter.
 173   if (USRLength >= LineRef.size() || ' ' != LineRef[USRLength])
 174     return false;
 175
 176   LookupName = LineRef.substr(0, USRLength);
 177   FilePath = LineRef.substr(USRLength + 1);
 178   return true;
 179 }
 180
 181 llvm::Expected<llvm::StringMap<std::string>>
 182 parseCrossTUIndex(StringRef IndexPath) {
 183   std::ifstream ExternalMapFile{std::string(IndexPath)};
 184   if (!ExternalMapFile)
 185     return llvm::make_error<IndexError>(index_error_code::missing_index_file,
 186                                         IndexPath.str());
 187
 188   llvm::StringMap<std::string> Result;
 189   std::string Line;
 190   unsigned LineNo = 1;
 191   while (std::getline(ExternalMapFile, Line)) {
 192     // Split lookup name and file path
 193     StringRef LookupName, FilePathInIndex;
 194     if (!parseCrossTUIndexItem(Line, LookupName, FilePathInIndex))
 195       return llvm::make_error<IndexError>(
 196           index_error_code::invalid_index_format, IndexPath.str(), LineNo);
 197
 198     // Store paths with posix-style directory separator.
 199     SmallString<32> FilePath(FilePathInIndex);
 200     llvm::sys::path::native(FilePath, llvm::sys::path::Style::posix);
 201
 202     bool InsertionOccured;
 203     std::tie(std::ignore, InsertionOccured) =
 204         Result.try_emplace(LookupName, FilePath.begin(), FilePath.end());
 205     if (!InsertionOccured)
 206       return llvm::make_error<IndexError>(
 207           index_error_code::multiple_definitions, IndexPath.str(), LineNo);
 208
 209     ++LineNo;
 210   }
 211   return Result;
 212 }
 213
 214 std::string
 215 createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
 216   std::ostringstream Result;
 217   for (const auto &E : Index)
 218     Result << E.getKey().size() << ':' << E.getKey().str() << ' '
 219            << E.getValue() << '\n';
 220   return Result.str();
 221 }
 222
 223 bool shouldImport(const VarDecl *VD, const ASTContext &ACtx) {
 224   CanQualType CT = ACtx.getCanonicalType(VD->getType());
 225   return CT.isConstQualified() && VD->getType().isTrivialType(ACtx);
 226 }
 227
 228 static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
 229   return D->hasBody(DefD);
 230 }
 231 static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
 232   return D->getAnyInitializer(DefD);
 233 }
 234 template <typename T> static bool hasBodyOrInit(const T *D) {
 235   const T *Unused;
 236   return hasBodyOrInit(D, Unused);
 237 }
 238
 239 CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
 240     : Context(CI.getASTContext()), ASTStorage(CI) {}
 241
 242 CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
 243
 244 llvm::Optional<std::string>
 245 CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
 246   SmallString<128> DeclUSR;
 247   bool Ret = index::generateUSRForDecl(ND, DeclUSR);
 248   if (Ret)
 249     return {};
 250   return std::string(DeclUSR.str());
 251 }
 252
 253 /// Recursively visits the decls of a DeclContext, and returns one with the
 254 /// given USR.
 255 template <typename T>
 256 const T *
 257 CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
 258                                                   StringRef LookupName) {
 259   assert(DC && "Declaration Context must not be null");
 260   for (const Decl *D : DC->decls()) {
 261     const auto *SubDC = dyn_cast<DeclContext>(D);
 262     if (SubDC)
 263       if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
 264         return ND;
 265
 266     const auto *ND = dyn_cast<T>(D);
 267     const T *ResultDecl;
 268     if (!ND || !hasBodyOrInit(ND, ResultDecl))
 269       continue;
 270     llvm::Optional<std::string> ResultLookupName = getLookupName(ResultDecl);
 271     if (!ResultLookupName || *ResultLookupName != LookupName)
 272       continue;
 273     return ResultDecl;
 274   }
 275   return nullptr;
 276 }
 277
 278 template <typename T>
 279 llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
 280     const T *D, StringRef CrossTUDir, StringRef IndexName,
 281     bool DisplayCTUProgress) {
 282   assert(D && "D is missing, bad call to this function!");
 283   assert(!hasBodyOrInit(D) &&
 284          "D has a body or init in current translation unit!");
 285   ++NumGetCTUCalled;
 286   const llvm::Optional<std::string> LookupName = getLookupName(D);
 287   if (!LookupName)
 288     return llvm::make_error<IndexError>(
 289         index_error_code::failed_to_generate_usr);
 290   llvm::Expected<ASTUnit *> ASTUnitOrError =
 291       loadExternalAST(*LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
 292   if (!ASTUnitOrError)
 293     return ASTUnitOrError.takeError();
 294   ASTUnit *Unit = *ASTUnitOrError;
 295   assert(&Unit->getFileManager() ==
 296          &Unit->getASTContext().getSourceManager().getFileManager());
 297
 298   const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
 299   const llvm::Triple &TripleFrom =
 300       Unit->getASTContext().getTargetInfo().getTriple();
 301   // The imported AST had been generated for a different target.
 302   // Some parts of the triple in the loaded ASTContext can be unknown while the
 303   // very same parts in the target ASTContext are known. Thus we check for the
 304   // known parts only.
 305   if (!hasEqualKnownFields(TripleTo, TripleFrom)) {
 306     // TODO: Pass the SourceLocation of the CallExpression for more precise
 307     // diagnostics.
 308     ++NumTripleMismatch;
 309     return llvm::make_error<IndexError>(index_error_code::triple_mismatch,
 310                                         std::string(Unit->getMainFileName()),
 311                                         TripleTo.str(), TripleFrom.str());
 312   }
 313
 314   const auto &LangTo = Context.getLangOpts();
 315   const auto &LangFrom = Unit->getASTContext().getLangOpts();
 316
 317   // FIXME: Currenty we do not support CTU across C++ and C and across
 318   // different dialects of C++.
 319   if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
 320     ++NumLangMismatch;
 321     return llvm::make_error<IndexError>(index_error_code::lang_mismatch);
 322   }
 323
 324   // If CPP dialects are different then return with error.
 325   //
 326   // Consider this STL code:
 327   //   template<typename _Alloc>
 328   //     struct __alloc_traits
 329   //   #if __cplusplus >= 201103L
 330   //     : std::allocator_traits<_Alloc>
 331   //   #endif
 332   //     { // ...
 333   //     };
 334   // This class template would create ODR errors during merging the two units,
 335   // since in one translation unit the class template has a base class, however
 336   // in the other unit it has none.
 337   if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
 338       LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
 339       LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
 340       LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
 341     ++NumLangDialectMismatch;
 342     return llvm::make_error<IndexError>(
 343         index_error_code::lang_dialect_mismatch);
 344   }
 345
 346   TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
 347   if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
 348     return importDefinition(ResultDecl, Unit);
 349   return llvm::make_error<IndexError>(index_error_code::failed_import);
 350 }
 351
 352 llvm::Expected<const FunctionDecl *>
 353 CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
 354                                                   StringRef CrossTUDir,
 355                                                   StringRef IndexName,
 356                                                   bool DisplayCTUProgress) {
 357   return getCrossTUDefinitionImpl(FD, CrossTUDir, IndexName,
 358                                   DisplayCTUProgress);
 359 }
 360
 361 llvm::Expected<const VarDecl *>
 362 CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
 363                                                   StringRef CrossTUDir,
 364                                                   StringRef IndexName,
 365                                                   bool DisplayCTUProgress) {
 366   return getCrossTUDefinitionImpl(VD, CrossTUDir, IndexName,
 367                                   DisplayCTUProgress);
 368 }
 369
 370 void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
 371   switch (IE.getCode()) {
 372   case index_error_code::missing_index_file:
 373     Context.getDiagnostics().Report(diag::err_ctu_error_opening)
 374         << IE.getFileName();
 375     break;
 376   case index_error_code::invalid_index_format:
 377     Context.getDiagnostics().Report(diag::err_extdefmap_parsing)
 378         << IE.getFileName() << IE.getLineNum();
 379     break;
 380   case index_error_code::multiple_definitions:
 381     Context.getDiagnostics().Report(diag::err_multiple_def_index)
 382         << IE.getLineNum();
 383     break;
 384   case index_error_code::triple_mismatch:
 385     Context.getDiagnostics().Report(diag::warn_ctu_incompat_triple)
 386         << IE.getFileName() << IE.getTripleToName() << IE.getTripleFromName();
 387     break;
 388   default:
 389     break;
 390   }
 391 }
 392
 393 CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
 394     CompilerInstance &CI)
 395     : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
 396              CI.getAnalyzerOpts()->CTUInvocationList),
 397       LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
 398                     ? CI.getAnalyzerOpts()->CTUImportCppThreshold
 399                     : CI.getAnalyzerOpts()->CTUImportThreshold) {}
 400
 401 llvm::Expected<ASTUnit *>
 402 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
 403     StringRef FileName, bool DisplayCTUProgress) {
 404   // Try the cache first.
 405   auto ASTCacheEntry = FileASTUnitMap.find(FileName);
 406   if (ASTCacheEntry == FileASTUnitMap.end()) {
 407
 408     // Do not load if the limit is reached.
 409     if (!LoadGuard) {
 410       ++NumASTLoadThresholdReached;
 411       return llvm::make_error<IndexError>(
 412           index_error_code::load_threshold_reached);
 413     }
 414
 415     auto LoadAttempt = Loader.load(FileName);
 416
 417     if (!LoadAttempt)
 418       return LoadAttempt.takeError();
 419
 420     std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
 421
 422     // Need the raw pointer and the unique_ptr as well.
 423     ASTUnit *Unit = LoadedUnit.get();
 424
 425     // Update the cache.
 426     FileASTUnitMap[FileName] = std::move(LoadedUnit);
 427
 428     LoadGuard.indicateLoadSuccess();
 429
 430     if (DisplayCTUProgress)
 431       llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
 432
 433     return Unit;
 434
 435   } else {
 436     // Found in the cache.
 437     return ASTCacheEntry->second.get();
 438   }
 439 }
 440
 441 llvm::Expected<ASTUnit *>
 442 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
 443     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
 444     bool DisplayCTUProgress) {
 445   // Try the cache first.
 446   auto ASTCacheEntry = NameASTUnitMap.find(FunctionName);
 447   if (ASTCacheEntry == NameASTUnitMap.end()) {
 448     // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
 449
 450     // Ensure that the Index is loaded, as we need to search in it.
 451     if (llvm::Error IndexLoadError =
 452             ensureCTUIndexLoaded(CrossTUDir, IndexName))
 453       return std::move(IndexLoadError);
 454
 455     // Check if there is an entry in the index for the function.
 456     if (!NameFileMap.count(FunctionName)) {
 457       ++NumNotInOtherTU;
 458       return llvm::make_error<IndexError>(index_error_code::missing_definition);
 459     }
 460
 461     // Search in the index for the filename where the definition of FunctionName
 462     // resides.
 463     if (llvm::Expected<ASTUnit *> FoundForFile =
 464             getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
 465
 466       // Update the cache.
 467       NameASTUnitMap[FunctionName] = *FoundForFile;
 468       return *FoundForFile;
 469
 470     } else {
 471       return FoundForFile.takeError();
 472     }
 473   } else {
 474     // Found in the cache.
 475     return ASTCacheEntry->second;
 476   }
 477 }
 478
 479 llvm::Expected<std::string>
 480 CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
 481     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
 482   if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
 483     return std::move(IndexLoadError);
 484   return NameFileMap[FunctionName];
 485 }
 486
 487 llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
 488     StringRef CrossTUDir, StringRef IndexName) {
 489   // Dont initialize if the map is filled.
 490   if (!NameFileMap.empty())
 491     return llvm::Error::success();
 492
 493   // Get the absolute path to the index file.
 494   SmallString<256> IndexFile = CrossTUDir;
 495   if (llvm::sys::path::is_absolute(IndexName))
 496     IndexFile = IndexName;
 497   else
 498     llvm::sys::path::append(IndexFile, IndexName);
 499
 500   if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
 501     // Initialize member map.
 502     NameFileMap = *IndexMapping;
 503     return llvm::Error::success();
 504   } else {
 505     // Error while parsing CrossTU index file.
 506     return IndexMapping.takeError();
 507   };
 508 }
 509
 510 llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
 511     StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
 512     bool DisplayCTUProgress) {
 513   // FIXME: The current implementation only supports loading decls with
 514   //        a lookup name from a single translation unit. If multiple
 515   //        translation units contains decls with the same lookup name an
 516   //        error will be returned.
 517
 518   // Try to get the value from the heavily cached storage.
 519   llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
 520       LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
 521
 522   if (!Unit)
 523     return Unit.takeError();
 524
 525   // Check whether the backing pointer of the Expected is a nullptr.
 526   if (!*Unit)
 527     return llvm::make_error<IndexError>(
 528         index_error_code::failed_to_get_external_ast);
 529
 530   return Unit;
 531 }
 532
 533 CrossTranslationUnitContext::ASTLoader::ASTLoader(
 534     CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
 535     : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
 536
 537 CrossTranslationUnitContext::LoadResultTy
 538 CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
 539   llvm::SmallString<256> Path;
 540   if (llvm::sys::path::is_absolute(Identifier, PathStyle)) {
 541     Path = Identifier;
 542   } else {
 543     Path = CTUDir;
 544     llvm::sys::path::append(Path, PathStyle, Identifier);
 545   }
 546
 547   // The path is stored in the InvocationList member in posix style. To
 548   // successfully lookup an entry based on filepath, it must be converted.
 549   llvm::sys::path::native(Path, PathStyle);
 550
 551   // Normalize by removing relative path components.
 552   llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle);
 553
 554   if (Path.endswith(".ast"))
 555     return loadFromDump(Path);
 556   else
 557     return loadFromSource(Path);
 558 }
 559
 560 CrossTranslationUnitContext::LoadResultTy
 561 CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
 562   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
 563   TextDiagnosticPrinter *DiagClient =
 564       new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
 565   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
 566   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
 567       new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
 568   return ASTUnit::LoadFromASTFile(
 569       std::string(ASTDumpPath.str()),
 570       CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
 571       Diags, CI.getFileSystemOpts());
 572 }
 573
 574 /// Load the AST from a source-file, which is supposed to be located inside the
 575 /// YAML formatted invocation list file under the filesystem path specified by
 576 /// \p InvocationList. The invocation list should contain absolute paths.
 577 /// \p SourceFilePath is the absolute path of the source file that contains the
 578 /// function definition the analysis is looking for. The Index is built by the
 579 /// \p clang-extdef-mapping tool, which is also supposed to be generating
 580 /// absolute paths.
 581 ///
 582 /// Proper diagnostic emission requires absolute paths, so even if a future
 583 /// change introduces the handling of relative paths, this must be taken into
 584 /// consideration.
 585 CrossTranslationUnitContext::LoadResultTy
 586 CrossTranslationUnitContext::ASTLoader::loadFromSource(
 587     StringRef SourceFilePath) {
 588
 589   if (llvm::Error InitError = lazyInitInvocationList())
 590     return std::move(InitError);
 591   assert(InvocationList);
 592
 593   auto Invocation = InvocationList->find(SourceFilePath);
 594   if (Invocation == InvocationList->end())
 595     return llvm::make_error<IndexError>(
 596         index_error_code::invocation_list_lookup_unsuccessful);
 597
 598   const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
 599
 600   SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
 601   std::transform(InvocationCommand.begin(), InvocationCommand.end(),
 602                  CommandLineArgs.begin(),
 603                  [](auto &&CmdPart) { return CmdPart.c_str(); });
 604
 605   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts{&CI.getDiagnosticOpts()};
 606   auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
 607   IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
 608       CI.getDiagnostics().getDiagnosticIDs()};
 609   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
 610       new DiagnosticsEngine{DiagID, &*DiagOpts, DiagClient});
 611
 612   return std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
 613       CommandLineArgs.begin(), (CommandLineArgs.end()),
 614       CI.getPCHContainerOperations(), Diags,
 615       CI.getHeaderSearchOpts().ResourceDir));
 616 }
 617
 618 llvm::Expected<InvocationListTy>
 619 parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle) {
 620   InvocationListTy InvocationList;
 621
 622   /// LLVM YAML parser is used to extract information from invocation list file.
 623   llvm::SourceMgr SM;
 624   llvm::yaml::Stream InvocationFile(FileContent, SM);
 625
 626   /// Only the first document is processed.
 627   llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
 628
 629   /// There has to be at least one document available.
 630   if (FirstInvocationFile == InvocationFile.end())
 631     return llvm::make_error<IndexError>(
 632         index_error_code::invocation_list_empty);
 633
 634   llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
 635   if (!DocumentRoot)
 636     return llvm::make_error<IndexError>(
 637         index_error_code::invocation_list_wrong_format);
 638
 639   /// According to the format specified the document must be a mapping, where
 640   /// the keys are paths to source files, and values are sequences of invocation
 641   /// parts.
 642   auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(DocumentRoot);
 643   if (!Mappings)
 644     return llvm::make_error<IndexError>(
 645         index_error_code::invocation_list_wrong_format);
 646
 647   for (auto &NextMapping : *Mappings) {
 648     /// The keys should be strings, which represent a source-file path.
 649     auto *Key = dyn_cast<llvm::yaml::ScalarNode>(NextMapping.getKey());
 650     if (!Key)
 651       return llvm::make_error<IndexError>(
 652           index_error_code::invocation_list_wrong_format);
 653
 654     SmallString<32> ValueStorage;
 655     StringRef SourcePath = Key->getValue(ValueStorage);
 656
 657     // Store paths with PathStyle directory separator.
 658     SmallString<32> NativeSourcePath(SourcePath);
 659     llvm::sys::path::native(NativeSourcePath, PathStyle);
 660
 661     StringRef InvocationKey = NativeSourcePath;
 662
 663     if (InvocationList.find(InvocationKey) != InvocationList.end())
 664       return llvm::make_error<IndexError>(
 665           index_error_code::invocation_list_ambiguous);
 666
 667     /// The values should be sequences of strings, each representing a part of
 668     /// the invocation.
 669     auto *Args = dyn_cast<llvm::yaml::SequenceNode>(NextMapping.getValue());
 670     if (!Args)
 671       return llvm::make_error<IndexError>(
 672           index_error_code::invocation_list_wrong_format);
 673
 674     for (auto &Arg : *Args) {
 675       auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(&Arg);
 676       if (!CmdString)
 677         return llvm::make_error<IndexError>(
 678             index_error_code::invocation_list_wrong_format);
 679       /// Every conversion starts with an empty working storage, as it is not
 680       /// clear if this is a requirement of the YAML parser.
 681       ValueStorage.clear();
 682       InvocationList[InvocationKey].emplace_back(
 683           CmdString->getValue(ValueStorage));
 684     }
 685
 686     if (InvocationList[InvocationKey].empty())
 687       return llvm::make_error<IndexError>(
 688           index_error_code::invocation_list_wrong_format);
 689   }
 690
 691   return InvocationList;
 692 }
 693
 694 llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
 695   /// Lazily initialize the invocation list member used for on-demand parsing.
 696   if (InvocationList)
 697     return llvm::Error::success();
 698   if (index_error_code::success != PreviousParsingResult)
 699     return llvm::make_error<IndexError>(PreviousParsingResult);
 700
 701   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
 702       llvm::MemoryBuffer::getFile(InvocationListFilePath);
 703   if (!FileContent) {
 704     PreviousParsingResult = index_error_code::invocation_list_file_not_found;
 705     return llvm::make_error<IndexError>(PreviousParsingResult);
 706   }
 707   std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
 708   assert(ContentBuffer && "If no error was produced after loading, the pointer "
 709                           "should not be nullptr.");
 710
 711   llvm::Expected<InvocationListTy> ExpectedInvocationList =
 712       parseInvocationList(ContentBuffer->getBuffer(), PathStyle);
 713
 714   // Handle the error to store the code for next call to this function.
 715   if (!ExpectedInvocationList) {
 716     llvm::handleAllErrors(
 717         ExpectedInvocationList.takeError(),
 718         [&](const IndexError &E) { PreviousParsingResult = E.getCode(); });
 719     return llvm::make_error<IndexError>(PreviousParsingResult);
 720   }
 721
 722   InvocationList = *ExpectedInvocationList;
 723
 724   return llvm::Error::success();
 725 }
 726
 727 template <typename T>
 728 llvm::Expected<const T *>
 729 CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
 730   assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
 731
 732   assert(&D->getASTContext() == &Unit->getASTContext() &&
 733          "ASTContext of Decl and the unit should match.");
 734   ASTImporter &Importer = getOrCreateASTImporter(Unit);
 735
 736   auto ToDeclOrError = Importer.Import(D);
 737   if (!ToDeclOrError) {
 738     handleAllErrors(ToDeclOrError.takeError(), [&](const ASTImportError &IE) {
 739       switch (IE.Error) {
 740       case ASTImportError::NameConflict:
 741         ++NumNameConflicts;
 742         break;
 743       case ASTImportError::UnsupportedConstruct:
 744         ++NumUnsupportedNodeFound;
 745         break;
 746       case ASTImportError::Unknown:
 747         llvm_unreachable("Unknown import error happened.");
 748         break;
 749       }
 750     });
 751     return llvm::make_error<IndexError>(index_error_code::failed_import);
 752   }
 753   auto *ToDecl = cast<T>(*ToDeclOrError);
 754   assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
 755   ++NumGetCTUSuccess;
 756
 757   // Parent map is invalidated after changing the AST.
 758   ToDecl->getASTContext().getParentMapContext().clear();
 759
 760   return ToDecl;
 761 }
 762
 763 llvm::Expected<const FunctionDecl *>
 764 CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
 765                                               ASTUnit *Unit) {
 766   return importDefinitionImpl(FD, Unit);
 767 }
 768
 769 llvm::Expected<const VarDecl *>
 770 CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
 771                                               ASTUnit *Unit) {
 772   return importDefinitionImpl(VD, Unit);
 773 }
 774
 775 void CrossTranslationUnitContext::lazyInitImporterSharedSt(
 776     TranslationUnitDecl *ToTU) {
 777   if (!ImporterSharedSt)
 778     ImporterSharedSt = std::make_shared<ASTImporterSharedState>(*ToTU);
 779 }
 780
 781 ASTImporter &
 782 CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
 783   ASTContext &From = Unit->getASTContext();
 784
 785   auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl());
 786   if (I != ASTUnitImporterMap.end())
 787     return *I->second;
 788   lazyInitImporterSharedSt(Context.getTranslationUnitDecl());
 789   ASTImporter *NewImporter = new ASTImporter(
 790       Context, Context.getSourceManager().getFileManager(), From,
 791       From.getSourceManager().getFileManager(), false, ImporterSharedSt);
 792   ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter);
 793   return *NewImporter;
 794 }
 795
 796 llvm::Optional<clang::MacroExpansionContext>
 797 CrossTranslationUnitContext::getMacroExpansionContextForSourceLocation(
 798     const clang::SourceLocation &ToLoc) const {
 799   // FIXME: Implement: Record such a context for every imported ASTUnit; lookup.
 800   return llvm::None;
 801 }
 802
 803 bool CrossTranslationUnitContext::isImportedAsNew(const Decl *ToDecl) const {
 804   if (!ImporterSharedSt)
 805     return false;
 806   return ImporterSharedSt->isNewDecl(const_cast<Decl *>(ToDecl));
 807 }
 808
 809 bool CrossTranslationUnitContext::hasError(const Decl *ToDecl) const {
 810   if (!ImporterSharedSt)
 811     return false;
 812   return static_cast<bool>(
 813       ImporterSharedSt->getImportDeclErrorIfAny(const_cast<Decl *>(ToDecl)));
 814 }
 815
 816 } // namespace cross_tu
 817 } // namespace clang