clang-tools-extra/clangd/ModulesBuilder.cpp

   1 //===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "ModulesBuilder.h"
  10 #include "Compiler.h"
  11 #include "support/Logger.h"
  12 #include "clang/Frontend/FrontendAction.h"
  13 #include "clang/Frontend/FrontendActions.h"
  14 #include "clang/Serialization/ASTReader.h"
  15
  16 namespace clang {
  17 namespace clangd {
  18
  19 namespace {
  20
  21 // Create a path to store module files. Generally it should be:
  22 //
  23 //   {TEMP_DIRS}/clangd/module_files/{hashed-file-name}-%%-%%-%%-%%-%%-%%/.
  24 //
  25 // {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp"
  26 // or "C:/TEMP".
  27 //
  28 // '%%' means random value to make the generated path unique.
  29 //
  30 // \param MainFile is used to get the root of the project from global
  31 // compilation database.
  32 //
  33 // TODO: Move these module fils out of the temporary directory if the module
  34 // files are persistent.
  35 llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) {
  36   llvm::SmallString<128> HashedPrefix = llvm::sys::path::filename(MainFile);
  37   // There might be multiple files with the same name in a project. So appending
  38   // the hash value of the full path to make sure they won't conflict.
  39   HashedPrefix += std::to_string(llvm::hash_value(MainFile));
  40
  41   llvm::SmallString<256> ResultPattern;
  42
  43   llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true,
  44                                          ResultPattern);
  45
  46   llvm::sys::path::append(ResultPattern, "clangd");
  47   llvm::sys::path::append(ResultPattern, "module_files");
  48
  49   llvm::sys::path::append(ResultPattern, HashedPrefix);
  50
  51   ResultPattern.append("-%%-%%-%%-%%-%%-%%");
  52
  53   llvm::SmallString<256> Result;
  54   llvm::sys::fs::createUniquePath(ResultPattern, Result,
  55                                   /*MakeAbsolute=*/false);
  56
  57   llvm::sys::fs::create_directories(Result);
  58   return Result;
  59 }
  60
  61 // Get a unique module file path under \param ModuleFilesPrefix.
  62 std::string getModuleFilePath(llvm::StringRef ModuleName,
  63                               PathRef ModuleFilesPrefix) {
  64   llvm::SmallString<256> ModuleFilePath(ModuleFilesPrefix);
  65   auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
  66   llvm::sys::path::append(ModuleFilePath, PrimaryModuleName);
  67   if (!PartitionName.empty()) {
  68     ModuleFilePath.append("-");
  69     ModuleFilePath.append(PartitionName);
  70   }
  71
  72   ModuleFilePath.append(".pcm");
  73   return std::string(ModuleFilePath);
  74 }
  75
  76 // FailedPrerequisiteModules - stands for the PrerequisiteModules which has
  77 // errors happened during the building process.
  78 class FailedPrerequisiteModules : public PrerequisiteModules {
  79 public:
  80   ~FailedPrerequisiteModules() override = default;
  81
  82   // We shouldn't adjust the compilation commands based on
  83   // FailedPrerequisiteModules.
  84   void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
  85   }
  86
  87   // FailedPrerequisiteModules can never be reused.
  88   bool
  89   canReuse(const CompilerInvocation &CI,
  90            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override {
  91     return false;
  92   }
  93 };
  94
  95 // StandalonePrerequisiteModules - stands for PrerequisiteModules for which all
  96 // the required modules are built successfully. All the module files
  97 // are owned by the StandalonePrerequisiteModules class.
  98 //
  99 // Any of the built module files won't be shared with other instances of the
 100 // class. So that we can avoid worrying thread safety.
 101 //
 102 // We don't need to worry about duplicated module names here since the standard
 103 // guarantees the module names should be unique to a program.
 104 class StandalonePrerequisiteModules : public PrerequisiteModules {
 105 public:
 106   StandalonePrerequisiteModules() = default;
 107
 108   StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete;
 109   StandalonePrerequisiteModules
 110   operator=(const StandalonePrerequisiteModules &) = delete;
 111   StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete;
 112   StandalonePrerequisiteModules
 113   operator=(StandalonePrerequisiteModules &&) = delete;
 114
 115   ~StandalonePrerequisiteModules() override = default;
 116
 117   void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
 118     // Appending all built module files.
 119     for (auto &RequiredModule : RequiredModules)
 120       Options.PrebuiltModuleFiles.insert_or_assign(
 121           RequiredModule.ModuleName, RequiredModule.ModuleFilePath);
 122   }
 123
 124   bool canReuse(const CompilerInvocation &CI,
 125                 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;
 126
 127   bool isModuleUnitBuilt(llvm::StringRef ModuleName) const {
 128     return BuiltModuleNames.contains(ModuleName);
 129   }
 130
 131   void addModuleFile(llvm::StringRef ModuleName,
 132                      llvm::StringRef ModuleFilePath) {
 133     RequiredModules.emplace_back(ModuleName, ModuleFilePath);
 134     BuiltModuleNames.insert(ModuleName);
 135   }
 136
 137 private:
 138   struct ModuleFile {
 139     ModuleFile(llvm::StringRef ModuleName, PathRef ModuleFilePath)
 140         : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}
 141
 142     ModuleFile(const ModuleFile &) = delete;
 143     ModuleFile operator=(const ModuleFile &) = delete;
 144
 145     // The move constructor is needed for llvm::SmallVector.
 146     ModuleFile(ModuleFile &&Other)
 147         : ModuleName(std::move(Other.ModuleName)),
 148           ModuleFilePath(std::move(Other.ModuleFilePath)) {}
 149
 150     ModuleFile &operator=(ModuleFile &&Other) = delete;
 151
 152     ~ModuleFile() {
 153       if (!ModuleFilePath.empty())
 154         llvm::sys::fs::remove(ModuleFilePath);
 155     }
 156
 157     std::string ModuleName;
 158     std::string ModuleFilePath;
 159   };
 160
 161   llvm::SmallVector<ModuleFile, 8> RequiredModules;
 162   // A helper class to speedup the query if a module is built.
 163   llvm::StringSet<> BuiltModuleNames;
 164 };
 165
 166 // Build a module file for module with `ModuleName`. The information of built
 167 // module file are stored in \param BuiltModuleFiles.
 168 llvm::Error buildModuleFile(llvm::StringRef ModuleName,
 169                             const GlobalCompilationDatabase &CDB,
 170                             const ThreadsafeFS &TFS, ProjectModules &MDB,
 171                             PathRef ModuleFilesPrefix,
 172                             StandalonePrerequisiteModules &BuiltModuleFiles) {
 173   if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
 174     return llvm::Error::success();
 175
 176   PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName);
 177   // It is possible that we're meeting third party modules (modules whose
 178   // source are not in the project. e.g, the std module may be a third-party
 179   // module for most projects) or something wrong with the implementation of
 180   // ProjectModules.
 181   // FIXME: How should we treat third party modules here? If we want to ignore
 182   // third party modules, we should return true instead of false here.
 183   // Currently we simply bail out.
 184   if (ModuleUnitFileName.empty())
 185     return llvm::createStringError("Failed to get the primary source");
 186
 187   // Try cheap operation earlier to boil-out cheaply if there are problems.
 188   auto Cmd = CDB.getCompileCommand(ModuleUnitFileName);
 189   if (!Cmd)
 190     return llvm::createStringError(
 191         llvm::formatv("No compile command for {0}", ModuleUnitFileName));
 192
 193   for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) {
 194     // Return early if there are errors building the module file.
 195     if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB,
 196                                           ModuleFilesPrefix, BuiltModuleFiles))
 197       return llvm::createStringError(
 198           llvm::formatv("Failed to build dependency {0}: {1}",
 199                         RequiredModuleName, llvm::toString(std::move(Err))));
 200   }
 201
 202   Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix);
 203
 204   ParseInputs Inputs;
 205   Inputs.TFS = &TFS;
 206   Inputs.CompileCommand = std::move(*Cmd);
 207
 208   IgnoreDiagnostics IgnoreDiags;
 209   auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
 210   if (!CI)
 211     return llvm::createStringError("Failed to build compiler invocation");
 212
 213   auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
 214   auto Buf = FS->getBufferForFile(Inputs.CompileCommand.Filename);
 215   if (!Buf)
 216     return llvm::createStringError("Failed to create buffer");
 217
 218   // In clang's driver, we will suppress the check for ODR violation in GMF.
 219   // See the implementation of RenderModulesOptions in Clang.cpp.
 220   CI->getLangOpts().SkipODRCheckInGMF = true;
 221
 222   // Hash the contents of input files and store the hash value to the BMI files.
 223   // So that we can check if the files are still valid when we want to reuse the
 224   // BMI files.
 225   CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
 226
 227   BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts());
 228
 229   CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
 230   auto Clang =
 231       prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
 232                               std::move(*Buf), std::move(FS), IgnoreDiags);
 233   if (!Clang)
 234     return llvm::createStringError("Failed to prepare compiler instance");
 235
 236   GenerateReducedModuleInterfaceAction Action;
 237   Clang->ExecuteAction(Action);
 238
 239   if (Clang->getDiagnostics().hasErrorOccurred())
 240     return llvm::createStringError("Compilation failed");
 241
 242   BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output);
 243   return llvm::Error::success();
 244 }
 245 } // namespace
 246
 247 std::unique_ptr<PrerequisiteModules>
 248 ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
 249                                             const ThreadsafeFS &TFS) const {
 250   std::unique_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
 251   if (!MDB) {
 252     elog("Failed to get Project Modules information for {0}", File);
 253     return std::make_unique<FailedPrerequisiteModules>();
 254   }
 255
 256   std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
 257   if (RequiredModuleNames.empty())
 258     return std::make_unique<StandalonePrerequisiteModules>();
 259
 260   llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File);
 261
 262   log("Trying to build required modules for {0} in {1}", File,
 263       ModuleFilesPrefix);
 264
 265   auto RequiredModules = std::make_unique<StandalonePrerequisiteModules>();
 266
 267   for (llvm::StringRef RequiredModuleName : RequiredModuleNames) {
 268     // Return early if there is any error.
 269     if (llvm::Error Err =
 270             buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(),
 271                             ModuleFilesPrefix, *RequiredModules.get())) {
 272       elog("Failed to build module {0}; due to {1}", RequiredModuleName,
 273            toString(std::move(Err)));
 274       return std::make_unique<FailedPrerequisiteModules>();
 275     }
 276   }
 277
 278   log("Built required modules for {0} in {1}", File, ModuleFilesPrefix);
 279
 280   return std::move(RequiredModules);
 281 }
 282
 283 bool StandalonePrerequisiteModules::canReuse(
 284     const CompilerInvocation &CI,
 285     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
 286   if (RequiredModules.empty())
 287     return true;
 288
 289   CompilerInstance Clang;
 290
 291   Clang.setInvocation(std::make_shared<CompilerInvocation>(CI));
 292   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
 293       CompilerInstance::createDiagnostics(new DiagnosticOptions());
 294   Clang.setDiagnostics(Diags.get());
 295
 296   FileManager *FM = Clang.createFileManager(VFS);
 297   Clang.createSourceManager(*FM);
 298
 299   if (!Clang.createTarget())
 300     return false;
 301
 302   assert(Clang.getHeaderSearchOptsPtr());
 303   adjustHeaderSearchOptions(Clang.getHeaderSearchOpts());
 304   // Since we don't need to compile the source code actually, the TU kind here
 305   // doesn't matter.
 306   Clang.createPreprocessor(TU_Complete);
 307   Clang.getHeaderSearchOpts().ForceCheckCXX20ModulesInputFiles = true;
 308   Clang.getHeaderSearchOpts().ValidateASTInputFilesContent = true;
 309
 310   // Following the practice of clang's driver to suppres the checking for ODR
 311   // violation in GMF.
 312   // See
 313   // https://clang.llvm.org/docs/StandardCPlusPlusModules.html#object-definition-consistency
 314   // for example.
 315   Clang.getLangOpts().SkipODRCheckInGMF = true;
 316
 317   Clang.createASTReader();
 318   for (auto &RequiredModule : RequiredModules) {
 319     llvm::StringRef BMIPath = RequiredModule.ModuleFilePath;
 320     // FIXME: Loading BMI fully is too heavy considering something cheaply to
 321     // check if we can reuse the BMI.
 322     auto ReadResult =
 323         Clang.getASTReader()->ReadAST(BMIPath, serialization::MK_MainFile,
 324                                       SourceLocation(), ASTReader::ARR_None);
 325
 326     if (ReadResult != ASTReader::Success) {
 327       elog("Can't reuse {0}: {1}", BMIPath, ReadResult);
 328       return false;
 329     }
 330   }
 331
 332   return true;
 333 }
 334
 335 } // namespace clangd
 336 } // namespace clang