clang-tools-extra/clang-doc/tool/ClangDocMain.cpp

   1 //===-- ClangDocMain.cpp - ClangDoc -----------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This tool for generating C and C++ documentation from source code
  10 // and comments. Generally, it runs a LibTooling FrontendAction on source files,
  11 // mapping each declaration in those files to its USR and serializing relevant
  12 // information into LLVM bitcode. It then runs a pass over the collected
  13 // declaration information, reducing by USR. There is an option to dump this
  14 // intermediate result to bitcode. Finally, it hands the reduced information
  15 // off to a generator, which does the final parsing from the intermediate
  16 // representation to the desired output format.
  17 //
  18 //===----------------------------------------------------------------------===//
  19
  20 #include "BitcodeReader.h"
  21 #include "BitcodeWriter.h"
  22 #include "ClangDoc.h"
  23 #include "Generators.h"
  24 #include "Representation.h"
  25 #include "clang/AST/AST.h"
  26 #include "clang/AST/Decl.h"
  27 #include "clang/ASTMatchers/ASTMatchFinder.h"
  28 #include "clang/ASTMatchers/ASTMatchersInternal.h"
  29 #include "clang/Driver/Options.h"
  30 #include "clang/Frontend/FrontendActions.h"
  31 #include "clang/Tooling/AllTUsExecution.h"
  32 #include "clang/Tooling/CommonOptionsParser.h"
  33 #include "clang/Tooling/Execution.h"
  34 #include "clang/Tooling/Tooling.h"
  35 #include "llvm/ADT/APFloat.h"
  36 #include "llvm/Support/CommandLine.h"
  37 #include "llvm/Support/Error.h"
  38 #include "llvm/Support/FileSystem.h"
  39 #include "llvm/Support/Mutex.h"
  40 #include "llvm/Support/Path.h"
  41 #include "llvm/Support/Process.h"
  42 #include "llvm/Support/Signals.h"
  43 #include "llvm/Support/ThreadPool.h"
  44 #include "llvm/Support/raw_ostream.h"
  45 #include <atomic>
  46 #include <mutex>
  47 #include <string>
  48
  49 using namespace clang::ast_matchers;
  50 using namespace clang::tooling;
  51 using namespace clang;
  52
  53 static llvm::cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
  54 static llvm::cl::OptionCategory ClangDocCategory("clang-doc options");
  55
  56 static llvm::cl::opt<std::string>
  57     ProjectName("project-name", llvm::cl::desc("Name of project."),
  58                 llvm::cl::cat(ClangDocCategory));
  59
  60 static llvm::cl::opt<bool> IgnoreMappingFailures(
  61     "ignore-map-errors",
  62     llvm::cl::desc("Continue if files are not mapped correctly."),
  63     llvm::cl::init(true), llvm::cl::cat(ClangDocCategory));
  64
  65 static llvm::cl::opt<std::string>
  66     OutDirectory("output",
  67                  llvm::cl::desc("Directory for outputting generated files."),
  68                  llvm::cl::init("docs"), llvm::cl::cat(ClangDocCategory));
  69
  70 static llvm::cl::opt<bool>
  71     PublicOnly("public", llvm::cl::desc("Document only public declarations."),
  72                llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
  73
  74 static llvm::cl::opt<bool> DoxygenOnly(
  75     "doxygen",
  76     llvm::cl::desc("Use only doxygen-style comments to generate docs."),
  77     llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
  78
  79 static llvm::cl::list<std::string> UserStylesheets(
  80     "stylesheets", llvm::cl::CommaSeparated,
  81     llvm::cl::desc("CSS stylesheets to extend the default styles."),
  82     llvm::cl::cat(ClangDocCategory));
  83
  84 static llvm::cl::opt<std::string> UserAssetPath(
  85     "asset",
  86     llvm::cl::desc("User supplied asset path to "
  87                    "override the default css and js files for html output"),
  88     llvm::cl::cat(ClangDocCategory));
  89
  90 static llvm::cl::opt<std::string> SourceRoot("source-root", llvm::cl::desc(R"(
  91 Directory where processed files are stored.
  92 Links to definition locations will only be
  93 generated if the file is in this dir.)"),
  94                                              llvm::cl::cat(ClangDocCategory));
  95
  96 static llvm::cl::opt<std::string>
  97     RepositoryUrl("repository", llvm::cl::desc(R"(
  98 URL of repository that hosts code.
  99 Used for links to definition locations.)"),
 100                   llvm::cl::cat(ClangDocCategory));
 101
 102 enum OutputFormatTy {
 103   md,
 104   yaml,
 105   html,
 106 };
 107
 108 static llvm::cl::opt<OutputFormatTy>
 109     FormatEnum("format", llvm::cl::desc("Format for outputted docs."),
 110                llvm::cl::values(clEnumValN(OutputFormatTy::yaml, "yaml",
 111                                            "Documentation in YAML format."),
 112                                 clEnumValN(OutputFormatTy::md, "md",
 113                                            "Documentation in MD format."),
 114                                 clEnumValN(OutputFormatTy::html, "html",
 115                                            "Documentation in HTML format.")),
 116                llvm::cl::init(OutputFormatTy::yaml),
 117                llvm::cl::cat(ClangDocCategory));
 118
 119 std::string getFormatString() {
 120   switch (FormatEnum) {
 121   case OutputFormatTy::yaml:
 122     return "yaml";
 123   case OutputFormatTy::md:
 124     return "md";
 125   case OutputFormatTy::html:
 126     return "html";
 127   }
 128   llvm_unreachable("Unknown OutputFormatTy");
 129 }
 130
 131 // This function isn't referenced outside its translation unit, but it
 132 // can't use the "static" keyword because its address is used for
 133 // GetMainExecutable (since some platforms don't support taking the
 134 // address of main, and some platforms can't implement GetMainExecutable
 135 // without being given the address of a function in the main executable).
 136 std::string getExecutablePath(const char *Argv0, void *MainAddr) {
 137   return llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
 138 }
 139
 140 llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) {
 141   using DirIt = llvm::sys::fs::directory_iterator;
 142   std::error_code FileErr;
 143   llvm::SmallString<128> FilePath(UserAssetPath);
 144   for (DirIt DirStart = DirIt(UserAssetPath, FileErr),
 145                    DirEnd;
 146        !FileErr && DirStart != DirEnd; DirStart.increment(FileErr)) {
 147     FilePath = DirStart->path();
 148     if (llvm::sys::fs::is_regular_file(FilePath)) {
 149       if (llvm::sys::path::extension(FilePath) == ".css")
 150         CDCtx.UserStylesheets.insert(CDCtx.UserStylesheets.begin(),
 151                                      std::string(FilePath));
 152       else if (llvm::sys::path::extension(FilePath) == ".js")
 153         CDCtx.JsScripts.emplace_back(FilePath.str());
 154     }
 155   }
 156   if (FileErr)
 157     return llvm::createFileError(FilePath, FileErr);
 158   return llvm::Error::success();
 159 }
 160
 161 llvm::Error getDefaultAssetFiles(const char *Argv0,
 162                                  clang::doc::ClangDocContext &CDCtx) {
 163   void *MainAddr = (void *)(intptr_t)getExecutablePath;
 164   std::string ClangDocPath = getExecutablePath(Argv0, MainAddr);
 165   llvm::SmallString<128> NativeClangDocPath;
 166   llvm::sys::path::native(ClangDocPath, NativeClangDocPath);
 167
 168   llvm::SmallString<128> AssetsPath;
 169   AssetsPath = llvm::sys::path::parent_path(NativeClangDocPath);
 170   llvm::sys::path::append(AssetsPath, "..", "share", "clang-doc");
 171   llvm::SmallString<128> DefaultStylesheet;
 172   llvm::sys::path::native(AssetsPath, DefaultStylesheet);
 173   llvm::sys::path::append(DefaultStylesheet,
 174                           "clang-doc-default-stylesheet.css");
 175   llvm::SmallString<128> IndexJS;
 176   llvm::sys::path::native(AssetsPath, IndexJS);
 177   llvm::sys::path::append(IndexJS, "index.js");
 178
 179   if (!llvm::sys::fs::is_regular_file(IndexJS))
 180     return llvm::createStringError(llvm::inconvertibleErrorCode(),
 181                                    "default index.js file missing at " +
 182                                        IndexJS + "\n");
 183
 184   if (!llvm::sys::fs::is_regular_file(DefaultStylesheet))
 185     return llvm::createStringError(
 186         llvm::inconvertibleErrorCode(),
 187         "default clang-doc-default-stylesheet.css file missing at " +
 188             DefaultStylesheet + "\n");
 189
 190   CDCtx.UserStylesheets.insert(CDCtx.UserStylesheets.begin(),
 191                                std::string(DefaultStylesheet));
 192   CDCtx.JsScripts.emplace_back(IndexJS.str());
 193
 194   return llvm::Error::success();
 195 }
 196
 197 llvm::Error getHtmlAssetFiles(const char *Argv0,
 198                               clang::doc::ClangDocContext &CDCtx) {
 199   if (!UserAssetPath.empty() &&
 200       !llvm::sys::fs::is_directory(std::string(UserAssetPath)))
 201     llvm::outs() << "Asset path supply is not a directory: " << UserAssetPath
 202                  << " falling back to default\n";
 203   if (llvm::sys::fs::is_directory(std::string(UserAssetPath)))
 204     return getAssetFiles(CDCtx);
 205   return getDefaultAssetFiles(Argv0, CDCtx);
 206 }
 207
 208 /// Make the output of clang-doc deterministic by sorting the children of
 209 /// namespaces and records.
 210 void sortUsrToInfo(llvm::StringMap<std::unique_ptr<doc::Info>> &USRToInfo) {
 211   for (auto &I : USRToInfo) {
 212     auto &Info = I.second;
 213     if (Info->IT == doc::InfoType::IT_namespace) {
 214       auto *Namespace = static_cast<clang::doc::NamespaceInfo *>(Info.get());
 215       Namespace->Children.sort();
 216     }
 217     if (Info->IT == doc::InfoType::IT_record) {
 218       auto *Record = static_cast<clang::doc::RecordInfo *>(Info.get());
 219       Record->Children.sort();
 220     }
 221   }
 222 }
 223
 224 int main(int argc, const char **argv) {
 225   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 226   std::error_code OK;
 227
 228   const char *Overview =
 229       R"(Generates documentation from source code and comments.
 230
 231 Example usage for files without flags (default):
 232
 233   $ clang-doc File1.cpp File2.cpp ... FileN.cpp
 234
 235 Example usage for a project using a compile commands database:
 236
 237   $ clang-doc --executor=all-TUs compile_commands.json
 238 )";
 239
 240   auto Executor = clang::tooling::createExecutorFromCommandLineArgs(
 241       argc, argv, ClangDocCategory, Overview);
 242
 243   if (!Executor) {
 244     llvm::errs() << toString(Executor.takeError()) << "\n";
 245     return 1;
 246   }
 247
 248   // Fail early if an invalid format was provided.
 249   std::string Format = getFormatString();
 250   llvm::outs() << "Emiting docs in " << Format << " format.\n";
 251   auto G = doc::findGeneratorByName(Format);
 252   if (!G) {
 253     llvm::errs() << toString(G.takeError()) << "\n";
 254     return 1;
 255   }
 256
 257   ArgumentsAdjuster ArgAdjuster;
 258   if (!DoxygenOnly)
 259     ArgAdjuster = combineAdjusters(
 260         getInsertArgumentAdjuster("-fparse-all-comments",
 261                                   tooling::ArgumentInsertPosition::END),
 262         ArgAdjuster);
 263
 264   clang::doc::ClangDocContext CDCtx = {
 265       Executor->get()->getExecutionContext(),
 266       ProjectName,
 267       PublicOnly,
 268       OutDirectory,
 269       SourceRoot,
 270       RepositoryUrl,
 271       {UserStylesheets.begin(), UserStylesheets.end()}
 272   };
 273
 274   if (Format == "html") {
 275     if (auto Err = getHtmlAssetFiles(argv[0], CDCtx)) {
 276       llvm::errs() << toString(std::move(Err)) << "\n";
 277       return 1;
 278     }
 279   }
 280
 281   // Mapping phase
 282   llvm::outs() << "Mapping decls...\n";
 283   auto Err =
 284       Executor->get()->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster);
 285   if (Err) {
 286     if (IgnoreMappingFailures)
 287       llvm::errs() << "Error mapping decls in files. Clang-doc will ignore "
 288                       "these files and continue:\n"
 289                    << toString(std::move(Err)) << "\n";
 290     else {
 291       llvm::errs() << toString(std::move(Err)) << "\n";
 292       return 1;
 293     }
 294   }
 295
 296   // Collect values into output by key.
 297   // In ToolResults, the Key is the hashed USR and the value is the
 298   // bitcode-encoded representation of the Info object.
 299   llvm::outs() << "Collecting infos...\n";
 300   llvm::StringMap<std::vector<StringRef>> USRToBitcode;
 301   Executor->get()->getToolResults()->forEachResult(
 302       [&](StringRef Key, StringRef Value) {
 303         USRToBitcode[Key].emplace_back(Value);
 304       });
 305
 306   // Collects all Infos according to their unique USR value. This map is added
 307   // to from the thread pool below and is protected by the USRToInfoMutex.
 308   llvm::sys::Mutex USRToInfoMutex;
 309   llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo;
 310
 311   // First reducing phase (reduce all decls into one info per decl).
 312   llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
 313   std::atomic<bool> Error;
 314   Error = false;
 315   llvm::sys::Mutex IndexMutex;
 316   // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
 317   llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency));
 318   for (auto &Group : USRToBitcode) {
 319     Pool.async([&]() {
 320       std::vector<std::unique_ptr<doc::Info>> Infos;
 321       for (auto &Bitcode : Group.getValue()) {
 322         llvm::BitstreamCursor Stream(Bitcode);
 323         doc::ClangDocBitcodeReader Reader(Stream);
 324         auto ReadInfos = Reader.readBitcode();
 325         if (!ReadInfos) {
 326           llvm::errs() << toString(ReadInfos.takeError()) << "\n";
 327           Error = true;
 328           return;
 329         }
 330         std::move(ReadInfos->begin(), ReadInfos->end(),
 331                   std::back_inserter(Infos));
 332       }
 333
 334       auto Reduced = doc::mergeInfos(Infos);
 335       if (!Reduced) {
 336         llvm::errs() << llvm::toString(Reduced.takeError());
 337         return;
 338       }
 339
 340       // Add a reference to this Info in the Index
 341       {
 342         std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex);
 343         clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced.get().get());
 344       }
 345
 346       // Save in the result map (needs a lock due to threaded access).
 347       {
 348         std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex);
 349         USRToInfo[Group.getKey()] = std::move(Reduced.get());
 350       }
 351     });
 352   }
 353
 354   Pool.wait();
 355
 356   if (Error)
 357     return 1;
 358
 359   sortUsrToInfo(USRToInfo);
 360
 361   // Ensure the root output directory exists.
 362   if (std::error_code Err = llvm::sys::fs::create_directories(OutDirectory);
 363       Err != std::error_code()) {
 364     llvm::errs() << "Failed to create directory '" << OutDirectory << "'\n";
 365     return 1;
 366   }
 367
 368   // Run the generator.
 369   llvm::outs() << "Generating docs...\n";
 370   if (auto Err =
 371           G->get()->generateDocs(OutDirectory, std::move(USRToInfo), CDCtx)) {
 372     llvm::errs() << toString(std::move(Err)) << "\n";
 373     return 1;
 374   }
 375
 376   llvm::outs() << "Generating assets for docs...\n";
 377   Err = G->get()->createResources(CDCtx);
 378   if (Err) {
 379     llvm::outs() << "warning: " << toString(std::move(Err)) << "\n";
 380   }
 381
 382   return 0;
 383 }