clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

   1 //===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements the ExtractAPIAction, and ASTVisitor/Consumer to
  11 /// collect API information.
  12 ///
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "TypedefUnderlyingTypeResolver.h"
  16 #include "clang/AST/ASTConsumer.h"
  17 #include "clang/AST/ASTContext.h"
  18 #include "clang/AST/Decl.h"
  19 #include "clang/AST/DeclCXX.h"
  20 #include "clang/AST/ParentMapContext.h"
  21 #include "clang/AST/RawCommentList.h"
  22 #include "clang/AST/RecursiveASTVisitor.h"
  23 #include "clang/Basic/SourceLocation.h"
  24 #include "clang/Basic/SourceManager.h"
  25 #include "clang/Basic/TargetInfo.h"
  26 #include "clang/ExtractAPI/API.h"
  27 #include "clang/ExtractAPI/AvailabilityInfo.h"
  28 #include "clang/ExtractAPI/DeclarationFragments.h"
  29 #include "clang/ExtractAPI/FrontendActions.h"
  30 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
  31 #include "clang/Frontend/ASTConsumers.h"
  32 #include "clang/Frontend/CompilerInstance.h"
  33 #include "clang/Frontend/FrontendOptions.h"
  34 #include "clang/Lex/MacroInfo.h"
  35 #include "clang/Lex/PPCallbacks.h"
  36 #include "clang/Lex/Preprocessor.h"
  37 #include "clang/Lex/PreprocessorOptions.h"
  38 #include "llvm/ADT/DenseSet.h"
  39 #include "llvm/ADT/STLExtras.h"
  40 #include "llvm/ADT/SmallVector.h"
  41 #include "llvm/Support/FileSystem.h"
  42 #include "llvm/Support/MemoryBuffer.h"
  43 #include "llvm/Support/Path.h"
  44 #include "llvm/Support/Regex.h"
  45 #include "llvm/Support/raw_ostream.h"
  46 #include <memory>
  47 #include <utility>
  48
  49 using namespace clang;
  50 using namespace extractapi;
  51
  52 namespace {
  53
  54 StringRef getTypedefName(const TagDecl *Decl) {
  55   if (const auto *TypedefDecl = Decl->getTypedefNameForAnonDecl())
  56     return TypedefDecl->getName();
  57
  58   return {};
  59 }
  60
  61 Optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
  62                                              StringRef File,
  63                                              bool *IsQuoted = nullptr) {
  64   assert(CI.hasFileManager() &&
  65          "CompilerInstance does not have a FileNamager!");
  66
  67   using namespace llvm::sys;
  68   // Matches framework include patterns
  69   const llvm::Regex Rule("/(.+)\\.framework/(.+)?Headers/(.+)");
  70
  71   const auto &FS = CI.getVirtualFileSystem();
  72
  73   SmallString<128> FilePath(File.begin(), File.end());
  74   FS.makeAbsolute(FilePath);
  75   path::remove_dots(FilePath, true);
  76   FilePath = path::convert_to_slash(FilePath);
  77   File = FilePath;
  78
  79   // Checks whether `Dir` is a strict path prefix of `File`. If so returns
  80   // the prefix length. Otherwise return 0.
  81   auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {
  82     llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());
  83     FS.makeAbsolute(DirPath);
  84     path::remove_dots(DirPath, true);
  85     Dir = DirPath;
  86     for (auto NI = path::begin(File), NE = path::end(File),
  87               DI = path::begin(Dir), DE = path::end(Dir);
  88          /*termination condition in loop*/; ++NI, ++DI) {
  89       // '.' components in File are ignored.
  90       while (NI != NE && *NI == ".")
  91         ++NI;
  92       if (NI == NE)
  93         break;
  94
  95       // '.' components in Dir are ignored.
  96       while (DI != DE && *DI == ".")
  97         ++DI;
  98
  99       // Dir is a prefix of File, up to '.' components and choice of path
 100       // separators.
 101       if (DI == DE)
 102         return NI - path::begin(File);
 103
 104       // Consider all path separators equal.
 105       if (NI->size() == 1 && DI->size() == 1 &&
 106           path::is_separator(NI->front()) && path::is_separator(DI->front()))
 107         continue;
 108
 109       // Special case Apple .sdk folders since the search path is typically a
 110       // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
 111       // located in `iPhoneSimulator.sdk` (the real folder).
 112       if (NI->endswith(".sdk") && DI->endswith(".sdk")) {
 113         StringRef NBasename = path::stem(*NI);
 114         StringRef DBasename = path::stem(*DI);
 115         if (DBasename.startswith(NBasename))
 116           continue;
 117       }
 118
 119       if (*NI != *DI)
 120         break;
 121     }
 122     return 0;
 123   };
 124
 125   unsigned PrefixLength = 0;
 126
 127   // Go through the search paths and find the first one that is a prefix of
 128   // the header.
 129   for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {
 130     // Note whether the match is found in a quoted entry.
 131     if (IsQuoted)
 132       *IsQuoted = Entry.Group == frontend::Quoted;
 133
 134     if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Entry.Path)) {
 135       if (auto HMap = HeaderMap::Create(*EntryFile, CI.getFileManager())) {
 136         // If this is a headermap entry, try to reverse lookup the full path
 137         // for a spelled name before mapping.
 138         StringRef SpelledFilename = HMap->reverseLookupFilename(File);
 139         if (!SpelledFilename.empty())
 140           return SpelledFilename.str();
 141
 142         // No matching mapping in this headermap, try next search entry.
 143         continue;
 144       }
 145     }
 146
 147     // Entry is a directory search entry, try to check if it's a prefix of File.
 148     PrefixLength = CheckDir(Entry.Path);
 149     if (PrefixLength > 0) {
 150       // The header is found in a framework path, construct the framework-style
 151       // include name `<Framework/Header.h>`
 152       if (Entry.IsFramework) {
 153         SmallVector<StringRef, 4> Matches;
 154         Rule.match(File, &Matches);
 155         // Returned matches are always in stable order.
 156         if (Matches.size() != 4)
 157           return None;
 158
 159         return path::convert_to_slash(
 160             (Matches[1].drop_front(Matches[1].rfind('/') + 1) + "/" +
 161              Matches[3])
 162                 .str());
 163       }
 164
 165       // The header is found in a normal search path, strip the search path
 166       // prefix to get an include name.
 167       return path::convert_to_slash(File.drop_front(PrefixLength));
 168     }
 169   }
 170
 171   // Couldn't determine a include name, use full path instead.
 172   return None;
 173 }
 174
 175 struct LocationFileChecker {
 176   bool isLocationInKnownFile(SourceLocation Loc) {
 177     // If the loc refers to a macro expansion we need to first get the file
 178     // location of the expansion.
 179     auto &SM = CI.getSourceManager();
 180     auto FileLoc = SM.getFileLoc(Loc);
 181     FileID FID = SM.getFileID(FileLoc);
 182     if (FID.isInvalid())
 183       return false;
 184
 185     const auto *File = SM.getFileEntryForID(FID);
 186     if (!File)
 187       return false;
 188
 189     if (KnownFileEntries.count(File))
 190       return true;
 191
 192     if (ExternalFileEntries.count(File))
 193       return false;
 194
 195     StringRef FileName = File->tryGetRealPathName().empty()
 196                              ? File->getName()
 197                              : File->tryGetRealPathName();
 198
 199     // Try to reduce the include name the same way we tried to include it.
 200     bool IsQuoted = false;
 201     if (auto IncludeName = getRelativeIncludeName(CI, FileName, &IsQuoted))
 202       if (llvm::any_of(KnownFiles,
 203                        [&IsQuoted, &IncludeName](const auto &KnownFile) {
 204                          return KnownFile.first.equals(*IncludeName) &&
 205                                 KnownFile.second == IsQuoted;
 206                        })) {
 207         KnownFileEntries.insert(File);
 208         return true;
 209       }
 210
 211     // Record that the file was not found to avoid future reverse lookup for
 212     // the same file.
 213     ExternalFileEntries.insert(File);
 214     return false;
 215   }
 216
 217   LocationFileChecker(const CompilerInstance &CI,
 218                       SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)
 219       : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {
 220     for (const auto &KnownFile : KnownFiles)
 221       if (auto FileEntry = CI.getFileManager().getFile(KnownFile.first))
 222         KnownFileEntries.insert(*FileEntry);
 223   }
 224
 225 private:
 226   const CompilerInstance &CI;
 227   SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;
 228   llvm::DenseSet<const FileEntry *> KnownFileEntries;
 229   llvm::DenseSet<const FileEntry *> ExternalFileEntries;
 230 };
 231
 232 /// The RecursiveASTVisitor to traverse symbol declarations and collect API
 233 /// information.
 234 class ExtractAPIVisitor : public RecursiveASTVisitor<ExtractAPIVisitor> {
 235 public:
 236   ExtractAPIVisitor(ASTContext &Context, LocationFileChecker &LCF, APISet &API)
 237       : Context(Context), API(API), LCF(LCF) {}
 238
 239   const APISet &getAPI() const { return API; }
 240
 241   bool VisitVarDecl(const VarDecl *Decl) {
 242     // Skip function parameters.
 243     if (isa<ParmVarDecl>(Decl))
 244       return true;
 245
 246     // Skip non-global variables in records (struct/union/class).
 247     if (Decl->getDeclContext()->isRecord())
 248       return true;
 249
 250     // Skip local variables inside function or method.
 251     if (!Decl->isDefinedOutsideFunctionOrMethod())
 252       return true;
 253
 254     // If this is a template but not specialization or instantiation, skip.
 255     if (Decl->getASTContext().getTemplateOrSpecializationInfo(Decl) &&
 256         Decl->getTemplateSpecializationKind() == TSK_Undeclared)
 257       return true;
 258
 259     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 260       return true;
 261
 262     // Collect symbol information.
 263     StringRef Name = Decl->getName();
 264     StringRef USR = API.recordUSR(Decl);
 265     PresumedLoc Loc =
 266         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 267     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
 268     DocComment Comment;
 269     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 270       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 271                                               Context.getDiagnostics());
 272
 273     // Build declaration fragments and sub-heading for the variable.
 274     DeclarationFragments Declaration =
 275         DeclarationFragmentsBuilder::getFragmentsForVar(Decl);
 276     DeclarationFragments SubHeading =
 277         DeclarationFragmentsBuilder::getSubHeading(Decl);
 278
 279     // Add the global variable record to the API set.
 280     API.addGlobalVar(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
 281                      Declaration, SubHeading);
 282     return true;
 283   }
 284
 285   bool VisitFunctionDecl(const FunctionDecl *Decl) {
 286     if (const auto *Method = dyn_cast<CXXMethodDecl>(Decl)) {
 287       // Skip member function in class templates.
 288       if (Method->getParent()->getDescribedClassTemplate() != nullptr)
 289         return true;
 290
 291       // Skip methods in records.
 292       for (auto P : Context.getParents(*Method)) {
 293         if (P.get<CXXRecordDecl>())
 294           return true;
 295       }
 296
 297       // Skip ConstructorDecl and DestructorDecl.
 298       if (isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method))
 299         return true;
 300     }
 301
 302     // Skip templated functions.
 303     switch (Decl->getTemplatedKind()) {
 304     case FunctionDecl::TK_NonTemplate:
 305     case FunctionDecl::TK_DependentNonTemplate:
 306       break;
 307     case FunctionDecl::TK_MemberSpecialization:
 308     case FunctionDecl::TK_FunctionTemplateSpecialization:
 309       if (auto *TemplateInfo = Decl->getTemplateSpecializationInfo()) {
 310         if (!TemplateInfo->isExplicitInstantiationOrSpecialization())
 311           return true;
 312       }
 313       break;
 314     case FunctionDecl::TK_FunctionTemplate:
 315     case FunctionDecl::TK_DependentFunctionTemplateSpecialization:
 316       return true;
 317     }
 318
 319     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 320       return true;
 321
 322     // Collect symbol information.
 323     StringRef Name = Decl->getName();
 324     StringRef USR = API.recordUSR(Decl);
 325     PresumedLoc Loc =
 326         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 327     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
 328     DocComment Comment;
 329     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 330       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 331                                               Context.getDiagnostics());
 332
 333     // Build declaration fragments, sub-heading, and signature of the function.
 334     DeclarationFragments Declaration =
 335         DeclarationFragmentsBuilder::getFragmentsForFunction(Decl);
 336     DeclarationFragments SubHeading =
 337         DeclarationFragmentsBuilder::getSubHeading(Decl);
 338     FunctionSignature Signature =
 339         DeclarationFragmentsBuilder::getFunctionSignature(Decl);
 340
 341     // Add the function record to the API set.
 342     API.addGlobalFunction(Name, USR, Loc, AvailabilitySet(Decl), Linkage,
 343                           Comment, Declaration, SubHeading, Signature);
 344     return true;
 345   }
 346
 347   bool VisitEnumDecl(const EnumDecl *Decl) {
 348     if (!Decl->isComplete())
 349       return true;
 350
 351     // Skip forward declaration.
 352     if (!Decl->isThisDeclarationADefinition())
 353       return true;
 354
 355     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 356       return true;
 357
 358     // Collect symbol information.
 359     std::string NameString = Decl->getQualifiedNameAsString();
 360     StringRef Name(NameString);
 361     if (Name.empty())
 362       Name = getTypedefName(Decl);
 363
 364     StringRef USR = API.recordUSR(Decl);
 365     PresumedLoc Loc =
 366         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 367     DocComment Comment;
 368     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 369       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 370                                               Context.getDiagnostics());
 371
 372     // Build declaration fragments and sub-heading for the enum.
 373     DeclarationFragments Declaration =
 374         DeclarationFragmentsBuilder::getFragmentsForEnum(Decl);
 375     DeclarationFragments SubHeading =
 376         DeclarationFragmentsBuilder::getSubHeading(Decl);
 377
 378     EnumRecord *EnumRecord =
 379         API.addEnum(API.copyString(Name), USR, Loc, AvailabilitySet(Decl),
 380                     Comment, Declaration, SubHeading);
 381
 382     // Now collect information about the enumerators in this enum.
 383     recordEnumConstants(EnumRecord, Decl->enumerators());
 384
 385     return true;
 386   }
 387
 388   bool VisitRecordDecl(const RecordDecl *Decl) {
 389     if (!Decl->isCompleteDefinition())
 390       return true;
 391
 392     // Skip C++ structs/classes/unions
 393     // TODO: support C++ records
 394     if (isa<CXXRecordDecl>(Decl))
 395       return true;
 396
 397     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 398       return true;
 399
 400     // Collect symbol information.
 401     StringRef Name = Decl->getName();
 402     if (Name.empty())
 403       Name = getTypedefName(Decl);
 404     StringRef USR = API.recordUSR(Decl);
 405     PresumedLoc Loc =
 406         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 407     DocComment Comment;
 408     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 409       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 410                                               Context.getDiagnostics());
 411
 412     // Build declaration fragments and sub-heading for the struct.
 413     DeclarationFragments Declaration =
 414         DeclarationFragmentsBuilder::getFragmentsForStruct(Decl);
 415     DeclarationFragments SubHeading =
 416         DeclarationFragmentsBuilder::getSubHeading(Decl);
 417
 418     StructRecord *StructRecord =
 419         API.addStruct(Name, USR, Loc, AvailabilitySet(Decl), Comment,
 420                       Declaration, SubHeading);
 421
 422     // Now collect information about the fields in this struct.
 423     recordStructFields(StructRecord, Decl->fields());
 424
 425     return true;
 426   }
 427
 428   bool VisitObjCInterfaceDecl(const ObjCInterfaceDecl *Decl) {
 429     // Skip forward declaration for classes (@class)
 430     if (!Decl->isThisDeclarationADefinition())
 431       return true;
 432
 433     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 434       return true;
 435
 436     // Collect symbol information.
 437     StringRef Name = Decl->getName();
 438     StringRef USR = API.recordUSR(Decl);
 439     PresumedLoc Loc =
 440         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 441     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
 442     DocComment Comment;
 443     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 444       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 445                                               Context.getDiagnostics());
 446
 447     // Build declaration fragments and sub-heading for the interface.
 448     DeclarationFragments Declaration =
 449         DeclarationFragmentsBuilder::getFragmentsForObjCInterface(Decl);
 450     DeclarationFragments SubHeading =
 451         DeclarationFragmentsBuilder::getSubHeading(Decl);
 452
 453     // Collect super class information.
 454     SymbolReference SuperClass;
 455     if (const auto *SuperClassDecl = Decl->getSuperClass()) {
 456       SuperClass.Name = SuperClassDecl->getObjCRuntimeNameAsString();
 457       SuperClass.USR = API.recordUSR(SuperClassDecl);
 458     }
 459
 460     ObjCInterfaceRecord *ObjCInterfaceRecord =
 461         API.addObjCInterface(Name, USR, Loc, AvailabilitySet(Decl), Linkage,
 462                              Comment, Declaration, SubHeading, SuperClass);
 463
 464     // Record all methods (selectors). This doesn't include automatically
 465     // synthesized property methods.
 466     recordObjCMethods(ObjCInterfaceRecord, Decl->methods());
 467     recordObjCProperties(ObjCInterfaceRecord, Decl->properties());
 468     recordObjCInstanceVariables(ObjCInterfaceRecord, Decl->ivars());
 469     recordObjCProtocols(ObjCInterfaceRecord, Decl->protocols());
 470
 471     return true;
 472   }
 473
 474   bool VisitObjCProtocolDecl(const ObjCProtocolDecl *Decl) {
 475     // Skip forward declaration for protocols (@protocol).
 476     if (!Decl->isThisDeclarationADefinition())
 477       return true;
 478
 479     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 480       return true;
 481
 482     // Collect symbol information.
 483     StringRef Name = Decl->getName();
 484     StringRef USR = API.recordUSR(Decl);
 485     PresumedLoc Loc =
 486         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 487     DocComment Comment;
 488     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 489       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 490                                               Context.getDiagnostics());
 491
 492     // Build declaration fragments and sub-heading for the protocol.
 493     DeclarationFragments Declaration =
 494         DeclarationFragmentsBuilder::getFragmentsForObjCProtocol(Decl);
 495     DeclarationFragments SubHeading =
 496         DeclarationFragmentsBuilder::getSubHeading(Decl);
 497
 498     ObjCProtocolRecord *ObjCProtocolRecord =
 499         API.addObjCProtocol(Name, USR, Loc, AvailabilitySet(Decl), Comment,
 500                             Declaration, SubHeading);
 501
 502     recordObjCMethods(ObjCProtocolRecord, Decl->methods());
 503     recordObjCProperties(ObjCProtocolRecord, Decl->properties());
 504     recordObjCProtocols(ObjCProtocolRecord, Decl->protocols());
 505
 506     return true;
 507   }
 508
 509   bool VisitTypedefNameDecl(const TypedefNameDecl *Decl) {
 510     // Skip ObjC Type Parameter for now.
 511     if (isa<ObjCTypeParamDecl>(Decl))
 512       return true;
 513
 514     if (!Decl->isDefinedOutsideFunctionOrMethod())
 515       return true;
 516
 517     if (!LCF.isLocationInKnownFile(Decl->getLocation()))
 518       return true;
 519
 520     PresumedLoc Loc =
 521         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 522     StringRef Name = Decl->getName();
 523     StringRef USR = API.recordUSR(Decl);
 524     DocComment Comment;
 525     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 526       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 527                                               Context.getDiagnostics());
 528
 529     QualType Type = Decl->getUnderlyingType();
 530     SymbolReference SymRef =
 531         TypedefUnderlyingTypeResolver(Context).getSymbolReferenceForType(Type,
 532                                                                          API);
 533
 534     API.addTypedef(Name, USR, Loc, AvailabilitySet(Decl), Comment,
 535                    DeclarationFragmentsBuilder::getFragmentsForTypedef(Decl),
 536                    DeclarationFragmentsBuilder::getSubHeading(Decl), SymRef);
 537
 538     return true;
 539   }
 540
 541   bool VisitObjCCategoryDecl(const ObjCCategoryDecl *Decl) {
 542     // Collect symbol information.
 543     StringRef Name = Decl->getName();
 544     StringRef USR = API.recordUSR(Decl);
 545     PresumedLoc Loc =
 546         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
 547     DocComment Comment;
 548     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
 549       Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 550                                               Context.getDiagnostics());
 551     // Build declaration fragments and sub-heading for the category.
 552     DeclarationFragments Declaration =
 553         DeclarationFragmentsBuilder::getFragmentsForObjCCategory(Decl);
 554     DeclarationFragments SubHeading =
 555         DeclarationFragmentsBuilder::getSubHeading(Decl);
 556
 557     const ObjCInterfaceDecl *InterfaceDecl = Decl->getClassInterface();
 558     SymbolReference Interface(InterfaceDecl->getName(),
 559                               API.recordUSR(InterfaceDecl));
 560
 561     ObjCCategoryRecord *ObjCCategoryRecord =
 562         API.addObjCCategory(Name, USR, Loc, AvailabilitySet(Decl), Comment,
 563                             Declaration, SubHeading, Interface);
 564
 565     recordObjCMethods(ObjCCategoryRecord, Decl->methods());
 566     recordObjCProperties(ObjCCategoryRecord, Decl->properties());
 567     recordObjCInstanceVariables(ObjCCategoryRecord, Decl->ivars());
 568     recordObjCProtocols(ObjCCategoryRecord, Decl->protocols());
 569
 570     return true;
 571   }
 572
 573 private:
 574   /// Collect API information for the enum constants and associate with the
 575   /// parent enum.
 576   void recordEnumConstants(EnumRecord *EnumRecord,
 577                            const EnumDecl::enumerator_range Constants) {
 578     for (const auto *Constant : Constants) {
 579       // Collect symbol information.
 580       StringRef Name = Constant->getName();
 581       StringRef USR = API.recordUSR(Constant);
 582       PresumedLoc Loc =
 583           Context.getSourceManager().getPresumedLoc(Constant->getLocation());
 584       DocComment Comment;
 585       if (auto *RawComment = Context.getRawCommentForDeclNoCache(Constant))
 586         Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 587                                                 Context.getDiagnostics());
 588
 589       // Build declaration fragments and sub-heading for the enum constant.
 590       DeclarationFragments Declaration =
 591           DeclarationFragmentsBuilder::getFragmentsForEnumConstant(Constant);
 592       DeclarationFragments SubHeading =
 593           DeclarationFragmentsBuilder::getSubHeading(Constant);
 594
 595       API.addEnumConstant(EnumRecord, Name, USR, Loc, AvailabilitySet(Constant),
 596                           Comment, Declaration, SubHeading);
 597     }
 598   }
 599
 600   /// Collect API information for the struct fields and associate with the
 601   /// parent struct.
 602   void recordStructFields(StructRecord *StructRecord,
 603                           const RecordDecl::field_range Fields) {
 604     for (const auto *Field : Fields) {
 605       // Collect symbol information.
 606       StringRef Name = Field->getName();
 607       StringRef USR = API.recordUSR(Field);
 608       PresumedLoc Loc =
 609           Context.getSourceManager().getPresumedLoc(Field->getLocation());
 610       DocComment Comment;
 611       if (auto *RawComment = Context.getRawCommentForDeclNoCache(Field))
 612         Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 613                                                 Context.getDiagnostics());
 614
 615       // Build declaration fragments and sub-heading for the struct field.
 616       DeclarationFragments Declaration =
 617           DeclarationFragmentsBuilder::getFragmentsForField(Field);
 618       DeclarationFragments SubHeading =
 619           DeclarationFragmentsBuilder::getSubHeading(Field);
 620
 621       API.addStructField(StructRecord, Name, USR, Loc, AvailabilitySet(Field),
 622                          Comment, Declaration, SubHeading);
 623     }
 624   }
 625
 626   /// Collect API information for the Objective-C methods and associate with the
 627   /// parent container.
 628   void recordObjCMethods(ObjCContainerRecord *Container,
 629                          const ObjCContainerDecl::method_range Methods) {
 630     for (const auto *Method : Methods) {
 631       // Don't record selectors for properties.
 632       if (Method->isPropertyAccessor())
 633         continue;
 634
 635       StringRef Name = API.copyString(Method->getSelector().getAsString());
 636       StringRef USR = API.recordUSR(Method);
 637       PresumedLoc Loc =
 638           Context.getSourceManager().getPresumedLoc(Method->getLocation());
 639       DocComment Comment;
 640       if (auto *RawComment = Context.getRawCommentForDeclNoCache(Method))
 641         Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 642                                                 Context.getDiagnostics());
 643
 644       // Build declaration fragments, sub-heading, and signature for the method.
 645       DeclarationFragments Declaration =
 646           DeclarationFragmentsBuilder::getFragmentsForObjCMethod(Method);
 647       DeclarationFragments SubHeading =
 648           DeclarationFragmentsBuilder::getSubHeading(Method);
 649       FunctionSignature Signature =
 650           DeclarationFragmentsBuilder::getFunctionSignature(Method);
 651
 652       API.addObjCMethod(Container, Name, USR, Loc, AvailabilitySet(Method),
 653                         Comment, Declaration, SubHeading, Signature,
 654                         Method->isInstanceMethod());
 655     }
 656   }
 657
 658   void recordObjCProperties(ObjCContainerRecord *Container,
 659                             const ObjCContainerDecl::prop_range Properties) {
 660     for (const auto *Property : Properties) {
 661       StringRef Name = Property->getName();
 662       StringRef USR = API.recordUSR(Property);
 663       PresumedLoc Loc =
 664           Context.getSourceManager().getPresumedLoc(Property->getLocation());
 665       DocComment Comment;
 666       if (auto *RawComment = Context.getRawCommentForDeclNoCache(Property))
 667         Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 668                                                 Context.getDiagnostics());
 669
 670       // Build declaration fragments and sub-heading for the property.
 671       DeclarationFragments Declaration =
 672           DeclarationFragmentsBuilder::getFragmentsForObjCProperty(Property);
 673       DeclarationFragments SubHeading =
 674           DeclarationFragmentsBuilder::getSubHeading(Property);
 675
 676       StringRef GetterName =
 677           API.copyString(Property->getGetterName().getAsString());
 678       StringRef SetterName =
 679           API.copyString(Property->getSetterName().getAsString());
 680
 681       // Get the attributes for property.
 682       unsigned Attributes = ObjCPropertyRecord::NoAttr;
 683       if (Property->getPropertyAttributes() &
 684           ObjCPropertyAttribute::kind_readonly)
 685         Attributes |= ObjCPropertyRecord::ReadOnly;
 686       if (Property->getPropertyAttributes() & ObjCPropertyAttribute::kind_class)
 687         Attributes |= ObjCPropertyRecord::Class;
 688
 689       API.addObjCProperty(
 690           Container, Name, USR, Loc, AvailabilitySet(Property), Comment,
 691           Declaration, SubHeading,
 692           static_cast<ObjCPropertyRecord::AttributeKind>(Attributes),
 693           GetterName, SetterName, Property->isOptional());
 694     }
 695   }
 696
 697   void recordObjCInstanceVariables(
 698       ObjCContainerRecord *Container,
 699       const llvm::iterator_range<
 700           DeclContext::specific_decl_iterator<ObjCIvarDecl>>
 701           Ivars) {
 702     for (const auto *Ivar : Ivars) {
 703       StringRef Name = Ivar->getName();
 704       StringRef USR = API.recordUSR(Ivar);
 705       PresumedLoc Loc =
 706           Context.getSourceManager().getPresumedLoc(Ivar->getLocation());
 707       DocComment Comment;
 708       if (auto *RawComment = Context.getRawCommentForDeclNoCache(Ivar))
 709         Comment = RawComment->getFormattedLines(Context.getSourceManager(),
 710                                                 Context.getDiagnostics());
 711
 712       // Build declaration fragments and sub-heading for the instance variable.
 713       DeclarationFragments Declaration =
 714           DeclarationFragmentsBuilder::getFragmentsForField(Ivar);
 715       DeclarationFragments SubHeading =
 716           DeclarationFragmentsBuilder::getSubHeading(Ivar);
 717
 718       ObjCInstanceVariableRecord::AccessControl Access =
 719           Ivar->getCanonicalAccessControl();
 720
 721       API.addObjCInstanceVariable(Container, Name, USR, Loc,
 722                                   AvailabilitySet(Ivar), Comment, Declaration,
 723                                   SubHeading, Access);
 724     }
 725   }
 726
 727   void recordObjCProtocols(ObjCContainerRecord *Container,
 728                            ObjCInterfaceDecl::protocol_range Protocols) {
 729     for (const auto *Protocol : Protocols)
 730       Container->Protocols.emplace_back(Protocol->getName(),
 731                                         API.recordUSR(Protocol));
 732   }
 733
 734   ASTContext &Context;
 735   APISet &API;
 736   LocationFileChecker &LCF;
 737 };
 738
 739 class ExtractAPIConsumer : public ASTConsumer {
 740 public:
 741   ExtractAPIConsumer(ASTContext &Context,
 742                      std::unique_ptr<LocationFileChecker> LCF, APISet &API)
 743       : Visitor(Context, *LCF, API), LCF(std::move(LCF)) {}
 744
 745   void HandleTranslationUnit(ASTContext &Context) override {
 746     // Use ExtractAPIVisitor to traverse symbol declarations in the context.
 747     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
 748   }
 749
 750 private:
 751   ExtractAPIVisitor Visitor;
 752   std::unique_ptr<LocationFileChecker> LCF;
 753 };
 754
 755 class MacroCallback : public PPCallbacks {
 756 public:
 757   MacroCallback(const SourceManager &SM, LocationFileChecker &LCF, APISet &API,
 758                 Preprocessor &PP)
 759       : SM(SM), LCF(LCF), API(API), PP(PP) {}
 760
 761   void MacroDefined(const Token &MacroNameToken,
 762                     const MacroDirective *MD) override {
 763     auto *MacroInfo = MD->getMacroInfo();
 764
 765     if (MacroInfo->isBuiltinMacro())
 766       return;
 767
 768     auto SourceLoc = MacroNameToken.getLocation();
 769     if (SM.isWrittenInBuiltinFile(SourceLoc) ||
 770         SM.isWrittenInCommandLineFile(SourceLoc))
 771       return;
 772
 773     PendingMacros.emplace_back(MacroNameToken, MD);
 774   }
 775
 776   // If a macro gets undefined at some point during preprocessing of the inputs
 777   // it means that it isn't an exposed API and we should therefore not add a
 778   // macro definition for it.
 779   void MacroUndefined(const Token &MacroNameToken, const MacroDefinition &MD,
 780                       const MacroDirective *Undef) override {
 781     // If this macro wasn't previously defined we don't need to do anything
 782     // here.
 783     if (!Undef)
 784       return;
 785
 786     llvm::erase_if(PendingMacros, [&MD, this](const PendingMacro &PM) {
 787       return MD.getMacroInfo()->isIdenticalTo(*PM.MD->getMacroInfo(), PP,
 788                                               /*Syntactically*/ false);
 789     });
 790   }
 791
 792   void EndOfMainFile() override {
 793     for (auto &PM : PendingMacros) {
 794       // `isUsedForHeaderGuard` is only set when the preprocessor leaves the
 795       // file so check for it here.
 796       if (PM.MD->getMacroInfo()->isUsedForHeaderGuard())
 797         continue;
 798
 799       if (!LCF.isLocationInKnownFile(PM.MacroNameToken.getLocation()))
 800         continue;
 801
 802       StringRef Name = PM.MacroNameToken.getIdentifierInfo()->getName();
 803       PresumedLoc Loc = SM.getPresumedLoc(PM.MacroNameToken.getLocation());
 804       StringRef USR =
 805           API.recordUSRForMacro(Name, PM.MacroNameToken.getLocation(), SM);
 806
 807       API.addMacroDefinition(
 808           Name, USR, Loc,
 809           DeclarationFragmentsBuilder::getFragmentsForMacro(Name, PM.MD),
 810           DeclarationFragmentsBuilder::getSubHeadingForMacro(Name));
 811     }
 812
 813     PendingMacros.clear();
 814   }
 815
 816 private:
 817   struct PendingMacro {
 818     Token MacroNameToken;
 819     const MacroDirective *MD;
 820
 821     PendingMacro(const Token &MacroNameToken, const MacroDirective *MD)
 822         : MacroNameToken(MacroNameToken), MD(MD) {}
 823   };
 824
 825   const SourceManager &SM;
 826   LocationFileChecker &LCF;
 827   APISet &API;
 828   Preprocessor &PP;
 829   llvm::SmallVector<PendingMacro> PendingMacros;
 830 };
 831
 832 } // namespace
 833
 834 std::unique_ptr<ASTConsumer>
 835 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 836   OS = CreateOutputFile(CI, InFile);
 837   if (!OS)
 838     return nullptr;
 839
 840   ProductName = CI.getFrontendOpts().ProductName;
 841
 842   // Now that we have enough information about the language options and the
 843   // target triple, let's create the APISet before anyone uses it.
 844   API = std::make_unique<APISet>(
 845       CI.getTarget().getTriple(),
 846       CI.getFrontendOpts().Inputs.back().getKind().getLanguage());
 847
 848   auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles);
 849
 850   CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
 851       CI.getSourceManager(), *LCF, *API, CI.getPreprocessor()));
 852
 853   return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),
 854                                               std::move(LCF), *API);
 855 }
 856
 857 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
 858   auto &Inputs = CI.getFrontendOpts().Inputs;
 859   if (Inputs.empty())
 860     return true;
 861
 862   if (!CI.hasFileManager())
 863     if (!CI.createFileManager())
 864       return false;
 865
 866   auto Kind = Inputs[0].getKind();
 867
 868   // Convert the header file inputs into a single input buffer.
 869   SmallString<256> HeaderContents;
 870   bool IsQuoted = false;
 871   for (const FrontendInputFile &FIF : Inputs) {
 872     if (Kind.isObjectiveC())
 873       HeaderContents += "#import";
 874     else
 875       HeaderContents += "#include";
 876
 877     StringRef FilePath = FIF.getFile();
 878     if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) {
 879       if (IsQuoted)
 880         HeaderContents += " \"";
 881       else
 882         HeaderContents += " <";
 883
 884       HeaderContents += *RelativeName;
 885
 886       if (IsQuoted)
 887         HeaderContents += "\"\n";
 888       else
 889         HeaderContents += ">\n";
 890       KnownInputFiles.emplace_back(static_cast<SmallString<32>>(*RelativeName),
 891                                    IsQuoted);
 892     } else {
 893       HeaderContents += " \"";
 894       HeaderContents += FilePath;
 895       HeaderContents += "\"\n";
 896       KnownInputFiles.emplace_back(FilePath, true);
 897     }
 898   }
 899
 900   if (CI.getHeaderSearchOpts().Verbose)
 901     CI.getVerboseOutputStream() << getInputBufferName() << ":\n"
 902                                 << HeaderContents << "\n";
 903
 904   Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
 905                                                 getInputBufferName());
 906
 907   // Set that buffer up as our "real" input in the CompilerInstance.
 908   Inputs.clear();
 909   Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);
 910
 911   return true;
 912 }
 913
 914 void ExtractAPIAction::EndSourceFileAction() {
 915   if (!OS)
 916     return;
 917
 918   // Setup a SymbolGraphSerializer to write out collected API information in
 919   // the Symbol Graph format.
 920   // FIXME: Make the kind of APISerializer configurable.
 921   SymbolGraphSerializer SGSerializer(*API, ProductName);
 922   SGSerializer.serialize(*OS);
 923   OS.reset();
 924 }
 925
 926 std::unique_ptr<raw_pwrite_stream>
 927 ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) {
 928   std::unique_ptr<raw_pwrite_stream> OS =
 929       CI.createDefaultOutputFile(/*Binary=*/false, InFile, /*Extension=*/"json",
 930                                  /*RemoveFileOnSignal=*/false);
 931   if (!OS)
 932     return nullptr;
 933   return OS;
 934 }