clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp

   1 //== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "GlobalCompilationDatabase.h"
  10 #include "index/Background.h"
  11 #include "support/Logger.h"
  12 #include "support/Path.h"
  13 #include "llvm/ADT/SmallString.h"
  14 #include "llvm/ADT/StringRef.h"
  15 #include "llvm/Support/Error.h"
  16 #include "llvm/Support/FileSystem.h"
  17 #include "llvm/Support/MemoryBuffer.h"
  18 #include "llvm/Support/Path.h"
  19 #include "llvm/Support/raw_ostream.h"
  20 #include <functional>
  21 #include <optional>
  22
  23 namespace clang {
  24 namespace clangd {
  25 namespace {
  26
  27 std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
  28                                      llvm::StringRef FilePath) {
  29   llvm::SmallString<128> ShardRootSS(ShardRoot);
  30   llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
  31                                            "." + llvm::toHex(digest(FilePath)) +
  32                                            ".idx");
  33   return std::string(ShardRootSS.str());
  34 }
  35
  36 // Uses disk as a storage for index shards.
  37 class DiskBackedIndexStorage : public BackgroundIndexStorage {
  38   std::string DiskShardRoot;
  39
  40 public:
  41   // Creates `DiskShardRoot` and any parents during construction.
  42   DiskBackedIndexStorage(llvm::StringRef Directory) : DiskShardRoot(Directory) {
  43     std::error_code OK;
  44     std::error_code EC = llvm::sys::fs::create_directories(DiskShardRoot);
  45     if (EC != OK) {
  46       elog("Failed to create directory {0} for index storage: {1}",
  47            DiskShardRoot, EC.message());
  48     }
  49   }
  50
  51   std::unique_ptr<IndexFileIn>
  52   loadShard(llvm::StringRef ShardIdentifier) const override {
  53     const std::string ShardPath =
  54         getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
  55     auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
  56     if (!Buffer)
  57       return nullptr;
  58     if (auto I =
  59             readIndexFile(Buffer->get()->getBuffer(), SymbolOrigin::Background))
  60       return std::make_unique<IndexFileIn>(std::move(*I));
  61     else
  62       elog("Error while reading shard {0}: {1}", ShardIdentifier,
  63            I.takeError());
  64     return nullptr;
  65   }
  66
  67   llvm::Error storeShard(llvm::StringRef ShardIdentifier,
  68                          IndexFileOut Shard) const override {
  69     auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
  70     return llvm::writeToOutput(ShardPath, [&Shard](llvm::raw_ostream &OS) {
  71       OS << Shard;
  72       return llvm::Error::success();
  73     });
  74   }
  75 };
  76
  77 // Doesn't persist index shards anywhere (used when the CDB dir is unknown).
  78 // We could consider indexing into ~/.clangd/ or so instead.
  79 class NullStorage : public BackgroundIndexStorage {
  80 public:
  81   std::unique_ptr<IndexFileIn>
  82   loadShard(llvm::StringRef ShardIdentifier) const override {
  83     return nullptr;
  84   }
  85
  86   llvm::Error storeShard(llvm::StringRef ShardIdentifier,
  87                          IndexFileOut Shard) const override {
  88     vlog("Couldn't find project for {0}, indexing in-memory only",
  89          ShardIdentifier);
  90     return llvm::Error::success();
  91   }
  92 };
  93
  94 // Creates and owns IndexStorages for multiple CDBs.
  95 // When a CDB root is found, shards are stored in $ROOT/.cache/clangd/index/.
  96 // When no root is found, the fallback path is ~/.cache/clangd/index/.
  97 class DiskBackedIndexStorageManager {
  98 public:
  99   DiskBackedIndexStorageManager(
 100       std::function<std::optional<ProjectInfo>(PathRef)> GetProjectInfo)
 101       : IndexStorageMapMu(std::make_unique<std::mutex>()),
 102         GetProjectInfo(std::move(GetProjectInfo)) {
 103     llvm::SmallString<128> FallbackDir;
 104     if (llvm::sys::path::cache_directory(FallbackDir))
 105       llvm::sys::path::append(FallbackDir, "clangd", "index");
 106     this->FallbackDir = FallbackDir.str().str();
 107   }
 108
 109   // Creates or fetches to storage from cache for the specified project.
 110   BackgroundIndexStorage *operator()(PathRef File) {
 111     std::lock_guard<std::mutex> Lock(*IndexStorageMapMu);
 112     llvm::SmallString<128> StorageDir(FallbackDir);
 113     if (auto PI = GetProjectInfo(File)) {
 114       StorageDir = PI->SourceRoot;
 115       llvm::sys::path::append(StorageDir, ".cache", "clangd", "index");
 116     }
 117     auto &IndexStorage = IndexStorageMap[StorageDir];
 118     if (!IndexStorage)
 119       IndexStorage = create(StorageDir);
 120     return IndexStorage.get();
 121   }
 122
 123 private:
 124   std::unique_ptr<BackgroundIndexStorage> create(PathRef CDBDirectory) {
 125     if (CDBDirectory.empty()) {
 126       elog("Tried to create storage for empty directory!");
 127       return std::make_unique<NullStorage>();
 128     }
 129     return std::make_unique<DiskBackedIndexStorage>(CDBDirectory);
 130   }
 131
 132   Path FallbackDir;
 133
 134   llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
 135   std::unique_ptr<std::mutex> IndexStorageMapMu;
 136
 137   std::function<std::optional<ProjectInfo>(PathRef)> GetProjectInfo;
 138 };
 139
 140 } // namespace
 141
 142 BackgroundIndexStorage::Factory
 143 BackgroundIndexStorage::createDiskBackedStorageFactory(
 144     std::function<std::optional<ProjectInfo>(PathRef)> GetProjectInfo) {
 145   return DiskBackedIndexStorageManager(std::move(GetProjectInfo));
 146 }
 147
 148 } // namespace clangd
 149 } // namespace clang