clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

   1 //===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements the ExtractAPIAction, and ASTConsumer to collect API
  11 /// information.
  12 ///
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "clang/AST/ASTConcept.h"
  16 #include "clang/AST/ASTConsumer.h"
  17 #include "clang/AST/ASTContext.h"
  18 #include "clang/AST/DeclObjC.h"
  19 #include "clang/Basic/DiagnosticFrontend.h"
  20 #include "clang/Basic/FileEntry.h"
  21 #include "clang/Basic/SourceLocation.h"
  22 #include "clang/Basic/SourceManager.h"
  23 #include "clang/Basic/TargetInfo.h"
  24 #include "clang/ExtractAPI/API.h"
  25 #include "clang/ExtractAPI/APIIgnoresList.h"
  26 #include "clang/ExtractAPI/ExtractAPIVisitor.h"
  27 #include "clang/ExtractAPI/FrontendActions.h"
  28 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
  29 #include "clang/Frontend/ASTConsumers.h"
  30 #include "clang/Frontend/CompilerInstance.h"
  31 #include "clang/Frontend/FrontendOptions.h"
  32 #include "clang/Frontend/MultiplexConsumer.h"
  33 #include "clang/Index/USRGeneration.h"
  34 #include "clang/InstallAPI/HeaderFile.h"
  35 #include "clang/Lex/MacroInfo.h"
  36 #include "clang/Lex/PPCallbacks.h"
  37 #include "clang/Lex/Preprocessor.h"
  38 #include "clang/Lex/PreprocessorOptions.h"
  39 #include "llvm/ADT/DenseSet.h"
  40 #include "llvm/ADT/STLExtras.h"
  41 #include "llvm/ADT/SmallString.h"
  42 #include "llvm/ADT/SmallVector.h"
  43 #include "llvm/ADT/StringRef.h"
  44 #include "llvm/Support/Casting.h"
  45 #include "llvm/Support/Error.h"
  46 #include "llvm/Support/FileSystem.h"
  47 #include "llvm/Support/MemoryBuffer.h"
  48 #include "llvm/Support/Path.h"
  49 #include "llvm/Support/Regex.h"
  50 #include "llvm/Support/raw_ostream.h"
  51 #include <memory>
  52 #include <optional>
  53 #include <utility>
  54
  55 using namespace clang;
  56 using namespace extractapi;
  57
  58 namespace {
  59
  60 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
  61                                                   StringRef File,
  62                                                   bool *IsQuoted = nullptr) {
  63   assert(CI.hasFileManager() &&
  64          "CompilerInstance does not have a FileNamager!");
  65
  66   using namespace llvm::sys;
  67   const auto &FS = CI.getVirtualFileSystem();
  68
  69   SmallString<128> FilePath(File.begin(), File.end());
  70   FS.makeAbsolute(FilePath);
  71   path::remove_dots(FilePath, true);
  72   FilePath = path::convert_to_slash(FilePath);
  73   File = FilePath;
  74
  75   // Checks whether `Dir` is a strict path prefix of `File`. If so returns
  76   // the prefix length. Otherwise return 0.
  77   auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {
  78     llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());
  79     FS.makeAbsolute(DirPath);
  80     path::remove_dots(DirPath, true);
  81     Dir = DirPath;
  82     for (auto NI = path::begin(File), NE = path::end(File),
  83               DI = path::begin(Dir), DE = path::end(Dir);
  84          /*termination condition in loop*/; ++NI, ++DI) {
  85       // '.' components in File are ignored.
  86       while (NI != NE && *NI == ".")
  87         ++NI;
  88       if (NI == NE)
  89         break;
  90
  91       // '.' components in Dir are ignored.
  92       while (DI != DE && *DI == ".")
  93         ++DI;
  94
  95       // Dir is a prefix of File, up to '.' components and choice of path
  96       // separators.
  97       if (DI == DE)
  98         return NI - path::begin(File);
  99
 100       // Consider all path separators equal.
 101       if (NI->size() == 1 && DI->size() == 1 &&
 102           path::is_separator(NI->front()) && path::is_separator(DI->front()))
 103         continue;
 104
 105       // Special case Apple .sdk folders since the search path is typically a
 106       // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
 107       // located in `iPhoneSimulator.sdk` (the real folder).
 108       if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) {
 109         StringRef NBasename = path::stem(*NI);
 110         StringRef DBasename = path::stem(*DI);
 111         if (DBasename.starts_with(NBasename))
 112           continue;
 113       }
 114
 115       if (*NI != *DI)
 116         break;
 117     }
 118     return 0;
 119   };
 120
 121   unsigned PrefixLength = 0;
 122
 123   // Go through the search paths and find the first one that is a prefix of
 124   // the header.
 125   for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {
 126     // Note whether the match is found in a quoted entry.
 127     if (IsQuoted)
 128       *IsQuoted = Entry.Group == frontend::Quoted;
 129
 130     if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Entry.Path)) {
 131       if (auto HMap = HeaderMap::Create(*EntryFile, CI.getFileManager())) {
 132         // If this is a headermap entry, try to reverse lookup the full path
 133         // for a spelled name before mapping.
 134         StringRef SpelledFilename = HMap->reverseLookupFilename(File);
 135         if (!SpelledFilename.empty())
 136           return SpelledFilename.str();
 137
 138         // No matching mapping in this headermap, try next search entry.
 139         continue;
 140       }
 141     }
 142
 143     // Entry is a directory search entry, try to check if it's a prefix of File.
 144     PrefixLength = CheckDir(Entry.Path);
 145     if (PrefixLength > 0) {
 146       // The header is found in a framework path, construct the framework-style
 147       // include name `<Framework/Header.h>`
 148       if (Entry.IsFramework) {
 149         SmallVector<StringRef, 4> Matches;
 150         clang::installapi::HeaderFile::getFrameworkIncludeRule().match(
 151             File, &Matches);
 152         // Returned matches are always in stable order.
 153         if (Matches.size() != 4)
 154           return std::nullopt;
 155
 156         return path::convert_to_slash(
 157             (Matches[1].drop_front(Matches[1].rfind('/') + 1) + "/" +
 158              Matches[3])
 159                 .str());
 160       }
 161
 162       // The header is found in a normal search path, strip the search path
 163       // prefix to get an include name.
 164       return path::convert_to_slash(File.drop_front(PrefixLength));
 165     }
 166   }
 167
 168   // Couldn't determine a include name, use full path instead.
 169   return std::nullopt;
 170 }
 171
 172 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
 173                                                   FileEntryRef FE,
 174                                                   bool *IsQuoted = nullptr) {
 175   return getRelativeIncludeName(CI, FE.getNameAsRequested(), IsQuoted);
 176 }
 177
 178 struct LocationFileChecker {
 179   bool operator()(SourceLocation Loc) {
 180     // If the loc refers to a macro expansion we need to first get the file
 181     // location of the expansion.
 182     auto &SM = CI.getSourceManager();
 183     auto FileLoc = SM.getFileLoc(Loc);
 184     FileID FID = SM.getFileID(FileLoc);
 185     if (FID.isInvalid())
 186       return false;
 187
 188     OptionalFileEntryRef File = SM.getFileEntryRefForID(FID);
 189     if (!File)
 190       return false;
 191
 192     if (KnownFileEntries.count(*File))
 193       return true;
 194
 195     if (ExternalFileEntries.count(*File))
 196       return false;
 197
 198     // Try to reduce the include name the same way we tried to include it.
 199     bool IsQuoted = false;
 200     if (auto IncludeName = getRelativeIncludeName(CI, *File, &IsQuoted))
 201       if (llvm::any_of(KnownFiles,
 202                        [&IsQuoted, &IncludeName](const auto &KnownFile) {
 203                          return KnownFile.first.equals(*IncludeName) &&
 204                                 KnownFile.second == IsQuoted;
 205                        })) {
 206         KnownFileEntries.insert(*File);
 207         return true;
 208       }
 209
 210     // Record that the file was not found to avoid future reverse lookup for
 211     // the same file.
 212     ExternalFileEntries.insert(*File);
 213     return false;
 214   }
 215
 216   LocationFileChecker(const CompilerInstance &CI,
 217                       SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)
 218       : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {
 219     for (const auto &KnownFile : KnownFiles)
 220       if (auto FE = CI.getFileManager().getOptionalFileRef(KnownFile.first))
 221         KnownFileEntries.insert(*FE);
 222   }
 223
 224 private:
 225   const CompilerInstance &CI;
 226   SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;
 227   llvm::DenseSet<const FileEntry *> KnownFileEntries;
 228   llvm::DenseSet<const FileEntry *> ExternalFileEntries;
 229 };
 230
 231 struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> {
 232   bool shouldDeclBeIncluded(const Decl *D) const {
 233     bool ShouldBeIncluded = true;
 234     // Check that we have the definition for redeclarable types.
 235     if (auto *TD = llvm::dyn_cast<TagDecl>(D))
 236       ShouldBeIncluded = TD->isThisDeclarationADefinition();
 237     else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(D))
 238       ShouldBeIncluded = Interface->isThisDeclarationADefinition();
 239     else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(D))
 240       ShouldBeIncluded = Protocol->isThisDeclarationADefinition();
 241
 242     ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation());
 243     return ShouldBeIncluded;
 244   }
 245
 246   BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context,
 247                          APISet &API)
 248       : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {}
 249
 250 private:
 251   LocationFileChecker &LCF;
 252 };
 253
 254 class WrappingExtractAPIConsumer : public ASTConsumer {
 255 public:
 256   WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)
 257       : Visitor(Context, API) {}
 258
 259   void HandleTranslationUnit(ASTContext &Context) override {
 260     // Use ExtractAPIVisitor to traverse symbol declarations in the context.
 261     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
 262   }
 263
 264 private:
 265   ExtractAPIVisitor<> Visitor;
 266 };
 267
 268 class ExtractAPIConsumer : public ASTConsumer {
 269 public:
 270   ExtractAPIConsumer(ASTContext &Context,
 271                      std::unique_ptr<LocationFileChecker> LCF, APISet &API)
 272       : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {}
 273
 274   void HandleTranslationUnit(ASTContext &Context) override {
 275     // Use ExtractAPIVisitor to traverse symbol declarations in the context.
 276     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
 277   }
 278
 279 private:
 280   BatchExtractAPIVisitor Visitor;
 281   std::unique_ptr<LocationFileChecker> LCF;
 282 };
 283
 284 class MacroCallback : public PPCallbacks {
 285 public:
 286   MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)
 287       : SM(SM), API(API), PP(PP) {}
 288
 289   void EndOfMainFile() override {
 290     for (const auto &M : PP.macros()) {
 291       auto *II = M.getFirst();
 292       auto MD = PP.getMacroDefinition(II);
 293       auto *MI = MD.getMacroInfo();
 294
 295       if (!MI)
 296         continue;
 297
 298       // Ignore header guard macros
 299       if (MI->isUsedForHeaderGuard())
 300         continue;
 301
 302       // Ignore builtin macros and ones defined via the command line.
 303       if (MI->isBuiltinMacro())
 304         continue;
 305
 306       auto DefLoc = MI->getDefinitionLoc();
 307
 308       if (SM.isWrittenInBuiltinFile(DefLoc) ||
 309           SM.isWrittenInCommandLineFile(DefLoc))
 310         continue;
 311
 312       auto AssociatedModuleMacros = MD.getModuleMacros();
 313       StringRef OwningModuleName;
 314       if (!AssociatedModuleMacros.empty())
 315         OwningModuleName = AssociatedModuleMacros.back()
 316                                ->getOwningModule()
 317                                ->getTopLevelModuleName();
 318
 319       if (!shouldMacroBeIncluded(DefLoc, OwningModuleName))
 320         continue;
 321
 322       StringRef Name = II->getName();
 323       PresumedLoc Loc = SM.getPresumedLoc(DefLoc);
 324       SmallString<128> USR;
 325       index::generateUSRForMacro(Name, DefLoc, SM, USR);
 326       API.createRecord<extractapi::MacroDefinitionRecord>(
 327           USR, Name, SymbolReference(), Loc,
 328           DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI),
 329           DeclarationFragmentsBuilder::getSubHeadingForMacro(Name),
 330           SM.isInSystemHeader(DefLoc));
 331     }
 332   }
 333
 334   virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
 335                                      StringRef ModuleName) {
 336     return true;
 337   }
 338
 339   const SourceManager &SM;
 340   APISet &API;
 341   Preprocessor &PP;
 342 };
 343
 344 class APIMacroCallback : public MacroCallback {
 345 public:
 346   APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,
 347                    LocationFileChecker &LCF)
 348       : MacroCallback(SM, API, PP), LCF(LCF) {}
 349
 350   bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
 351                              StringRef ModuleName) override {
 352     // Do not include macros from external files
 353     return LCF(MacroLoc);
 354   }
 355
 356 private:
 357   LocationFileChecker &LCF;
 358 };
 359
 360 std::unique_ptr<llvm::raw_pwrite_stream>
 361 createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) {
 362   auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir;
 363
 364   SmallString<256> FileName;
 365   llvm::sys::path::append(FileName, OutputDirectory,
 366                           BaseName + ".symbols.json");
 367   return CI.createOutputFile(
 368       FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false,
 369       /*UseTemporary*/ true, /*CreateMissingDirectories*/ true);
 370 }
 371
 372 } // namespace
 373
 374 void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) {
 375   SymbolGraphSerializerOption SerializationOptions;
 376   SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs;
 377   SerializationOptions.EmitSymbolLabelsForTesting =
 378       CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting;
 379
 380   if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) {
 381     auto ConstructOutputFile = [&CI](Twine BaseName) {
 382       return createAdditionalSymbolGraphFile(CI, BaseName);
 383     };
 384
 385     SymbolGraphSerializer::serializeWithExtensionGraphs(
 386         *OS, *API, IgnoresList, ConstructOutputFile, SerializationOptions);
 387   } else {
 388     SymbolGraphSerializer::serializeMainSymbolGraph(*OS, *API, IgnoresList,
 389                                                     SerializationOptions);
 390   }
 391
 392   // Flush the stream and close the main output stream.
 393   OS.reset();
 394 }
 395
 396 std::unique_ptr<ASTConsumer>
 397 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 398   auto ProductName = CI.getFrontendOpts().ProductName;
 399
 400   if (CI.getFrontendOpts().SymbolGraphOutputDir.empty())
 401     OS = CI.createDefaultOutputFile(/*Binary*/ false, InFile,
 402                                     /*Extension*/ "symbols.json",
 403                                     /*RemoveFileOnSignal*/ false,
 404                                     /*CreateMissingDirectories*/ true);
 405   else
 406     OS = createAdditionalSymbolGraphFile(CI, ProductName);
 407
 408   if (!OS)
 409     return nullptr;
 410
 411   // Now that we have enough information about the language options and the
 412   // target triple, let's create the APISet before anyone uses it.
 413   API = std::make_unique<APISet>(
 414       CI.getTarget().getTriple(),
 415       CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);
 416
 417   auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles);
 418
 419   CI.getPreprocessor().addPPCallbacks(std::make_unique<APIMacroCallback>(
 420       CI.getSourceManager(), *API, CI.getPreprocessor(), *LCF));
 421
 422   // Do not include location in anonymous decls.
 423   PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
 424   Policy.AnonymousTagLocations = false;
 425   CI.getASTContext().setPrintingPolicy(Policy);
 426
 427   if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
 428     llvm::handleAllErrors(
 429         APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
 430                                CI.getFileManager())
 431             .moveInto(IgnoresList),
 432         [&CI](const IgnoresFileNotFound &Err) {
 433           CI.getDiagnostics().Report(
 434               diag::err_extract_api_ignores_file_not_found)
 435               << Err.Path;
 436         });
 437   }
 438
 439   return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),
 440                                               std::move(LCF), *API);
 441 }
 442
 443 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
 444   auto &Inputs = CI.getFrontendOpts().Inputs;
 445   if (Inputs.empty())
 446     return true;
 447
 448   if (!CI.hasFileManager())
 449     if (!CI.createFileManager())
 450       return false;
 451
 452   auto Kind = Inputs[0].getKind();
 453
 454   // Convert the header file inputs into a single input buffer.
 455   SmallString<256> HeaderContents;
 456   bool IsQuoted = false;
 457   for (const FrontendInputFile &FIF : Inputs) {
 458     if (Kind.isObjectiveC())
 459       HeaderContents += "#import";
 460     else
 461       HeaderContents += "#include";
 462
 463     StringRef FilePath = FIF.getFile();
 464     if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) {
 465       if (IsQuoted)
 466         HeaderContents += " \"";
 467       else
 468         HeaderContents += " <";
 469
 470       HeaderContents += *RelativeName;
 471
 472       if (IsQuoted)
 473         HeaderContents += "\"\n";
 474       else
 475         HeaderContents += ">\n";
 476       KnownInputFiles.emplace_back(static_cast<SmallString<32>>(*RelativeName),
 477                                    IsQuoted);
 478     } else {
 479       HeaderContents += " \"";
 480       HeaderContents += FilePath;
 481       HeaderContents += "\"\n";
 482       KnownInputFiles.emplace_back(FilePath, true);
 483     }
 484   }
 485
 486   if (CI.getHeaderSearchOpts().Verbose)
 487     CI.getVerboseOutputStream() << getInputBufferName() << ":\n"
 488                                 << HeaderContents << "\n";
 489
 490   Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
 491                                                 getInputBufferName());
 492
 493   // Set that buffer up as our "real" input in the CompilerInstance.
 494   Inputs.clear();
 495   Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);
 496
 497   return true;
 498 }
 499
 500 void ExtractAPIAction::EndSourceFileAction() {
 501   ImplEndSourceFileAction(getCompilerInstance());
 502 }
 503
 504 std::unique_ptr<ASTConsumer>
 505 WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,
 506                                             StringRef InFile) {
 507   auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
 508   if (!OtherConsumer)
 509     return nullptr;
 510
 511   CreatedASTConsumer = true;
 512
 513   ProductName = CI.getFrontendOpts().ProductName;
 514   auto InputFilename = llvm::sys::path::filename(InFile);
 515   OS = createAdditionalSymbolGraphFile(CI, InputFilename);
 516
 517   // Now that we have enough information about the language options and the
 518   // target triple, let's create the APISet before anyone uses it.
 519   API = std::make_unique<APISet>(
 520       CI.getTarget().getTriple(),
 521       CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);
 522
 523   CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
 524       CI.getSourceManager(), *API, CI.getPreprocessor()));
 525
 526   // Do not include location in anonymous decls.
 527   PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
 528   Policy.AnonymousTagLocations = false;
 529   CI.getASTContext().setPrintingPolicy(Policy);
 530
 531   if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
 532     llvm::handleAllErrors(
 533         APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
 534                                CI.getFileManager())
 535             .moveInto(IgnoresList),
 536         [&CI](const IgnoresFileNotFound &Err) {
 537           CI.getDiagnostics().Report(
 538               diag::err_extract_api_ignores_file_not_found)
 539               << Err.Path;
 540         });
 541   }
 542
 543   auto WrappingConsumer =
 544       std::make_unique<WrappingExtractAPIConsumer>(CI.getASTContext(), *API);
 545   std::vector<std::unique_ptr<ASTConsumer>> Consumers;
 546   Consumers.push_back(std::move(OtherConsumer));
 547   Consumers.push_back(std::move(WrappingConsumer));
 548
 549   return std::make_unique<MultiplexConsumer>(std::move(Consumers));
 550 }
 551
 552 void WrappingExtractAPIAction::EndSourceFileAction() {
 553   // Invoke wrapped action's method.
 554   WrapperFrontendAction::EndSourceFileAction();
 555
 556   if (CreatedASTConsumer) {
 557     ImplEndSourceFileAction(getCompilerInstance());
 558   }
 559 }