llvm/lib/Support/FileCollector.cpp

   1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Support/FileCollector.h"
  10 #include "llvm/ADT/SmallString.h"
  11 #include "llvm/ADT/Twine.h"
  12 #include "llvm/Support/FileSystem.h"
  13 #include "llvm/Support/Path.h"
  14 #include "llvm/Support/Process.h"
  15
  16 using namespace llvm;
  17
  18 FileCollectorBase::FileCollectorBase() = default;
  19 FileCollectorBase::~FileCollectorBase() = default;
  20
  21 void FileCollectorBase::addFile(const Twine &File) {
  22   std::lock_guard<std::mutex> lock(Mutex);
  23   std::string FileStr = File.str();
  24   if (markAsSeen(FileStr))
  25     addFileImpl(FileStr);
  26 }
  27
  28 void FileCollectorBase::addDirectory(const Twine &Dir) {
  29   assert(sys::fs::is_directory(Dir));
  30   std::error_code EC;
  31   addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC);
  32 }
  33
  34 static bool isCaseSensitivePath(StringRef Path) {
  35   SmallString<256> TmpDest = Path, UpperDest, RealDest;
  36
  37   // Remove component traversals, links, etc.
  38   if (sys::fs::real_path(Path, TmpDest))
  39     return true; // Current default value in vfs.yaml
  40   Path = TmpDest;
  41
  42   // Change path to all upper case and ask for its real path, if the latter
  43   // exists and is equal to path, it's not case sensitive. Default to case
  44   // sensitive in the absence of real_path, since this is the YAMLVFSWriter
  45   // default.
  46   UpperDest = Path.upper();
  47   if (!sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
  48     return false;
  49   return true;
  50 }
  51
  52 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
  53     : Root(Root), OverlayRoot(OverlayRoot) {
  54   assert(sys::path::is_absolute(Root) && "Root not absolute");
  55   assert(sys::path::is_absolute(OverlayRoot) && "OverlayRoot not absolute");
  56 }
  57
  58 void FileCollector::PathCanonicalizer::updateWithRealPath(
  59     SmallVectorImpl<char> &Path) {
  60   StringRef SrcPath(Path.begin(), Path.size());
  61   StringRef Filename = sys::path::filename(SrcPath);
  62   StringRef Directory = sys::path::parent_path(SrcPath);
  63
  64   // Use real_path to fix any symbolic link component present in the directory
  65   // part of the path, caching the search because computing the real path is
  66   // expensive.
  67   SmallString<256> RealPath;
  68   auto DirWithSymlink = CachedDirs.find(Directory);
  69   if (DirWithSymlink == CachedDirs.end()) {
  70     // FIXME: Should this be a call to FileSystem::getRealpath(), in some
  71     // cases? What if there is nothing on disk?
  72     if (sys::fs::real_path(Directory, RealPath))
  73       return;
  74     CachedDirs[Directory] = std::string(RealPath);
  75   } else {
  76     RealPath = DirWithSymlink->second;
  77   }
  78
  79   // Finish recreating the path by appending the original filename, since we
  80   // don't need to resolve symlinks in the filename.
  81   //
  82   // FIXME: If we can cope with this, maybe we can cope without calling
  83   // getRealPath() at all when there's no ".." component.
  84   sys::path::append(RealPath, Filename);
  85
  86   // Swap to create the output.
  87   Path.swap(RealPath);
  88 }
  89
  90 /// Make Path absolute.
  91 static void makeAbsolute(SmallVectorImpl<char> &Path) {
  92   // We need an absolute src path to append to the root.
  93   sys::fs::make_absolute(Path);
  94
  95   // Canonicalize src to a native path to avoid mixed separator styles.
  96   sys::path::native(Path);
  97
  98   // Remove redundant leading "./" pieces and consecutive separators.
  99   Path.erase(Path.begin(), sys::path::remove_leading_dotslash(
 100                                StringRef(Path.begin(), Path.size()))
 101                                .begin());
 102 }
 103
 104 FileCollector::PathCanonicalizer::PathStorage
 105 FileCollector::PathCanonicalizer::canonicalize(StringRef SrcPath) {
 106   PathStorage Paths;
 107   Paths.VirtualPath = SrcPath;
 108   makeAbsolute(Paths.VirtualPath);
 109
 110   // If a ".." component is present after a symlink component, remove_dots may
 111   // lead to the wrong real destination path. Let the source be canonicalized
 112   // like that but make sure we always use the real path for the destination.
 113   Paths.CopyFrom = Paths.VirtualPath;
 114   updateWithRealPath(Paths.CopyFrom);
 115
 116   // Canonicalize the virtual path by removing "..", "." components.
 117   sys::path::remove_dots(Paths.VirtualPath, /*remove_dot_dot=*/true);
 118
 119   return Paths;
 120 }
 121
 122 void FileCollector::addFileImpl(StringRef SrcPath) {
 123   PathCanonicalizer::PathStorage Paths = Canonicalizer.canonicalize(SrcPath);
 124
 125   SmallString<256> DstPath = StringRef(Root);
 126   sys::path::append(DstPath, sys::path::relative_path(Paths.CopyFrom));
 127
 128   // Always map a canonical src path to its real path into the YAML, by doing
 129   // this we map different virtual src paths to the same entry in the VFS
 130   // overlay, which is a way to emulate symlink inside the VFS; this is also
 131   // needed for correctness, not doing that can lead to module redefinition
 132   // errors.
 133   addFileToMapping(Paths.VirtualPath, DstPath);
 134 }
 135
 136 llvm::vfs::directory_iterator
 137 FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
 138                                 IntrusiveRefCntPtr<vfs::FileSystem> FS,
 139                                 std::error_code &EC) {
 140   auto It = FS->dir_begin(Dir, EC);
 141   if (EC)
 142     return It;
 143   addFile(Dir);
 144   for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
 145     if (It->type() == sys::fs::file_type::regular_file ||
 146         It->type() == sys::fs::file_type::directory_file ||
 147         It->type() == sys::fs::file_type::symlink_file) {
 148       addFile(It->path());
 149     }
 150   }
 151   if (EC)
 152     return It;
 153   // Return a new iterator.
 154   return FS->dir_begin(Dir, EC);
 155 }
 156
 157 /// Set the access and modification time for the given file from the given
 158 /// status object.
 159 static std::error_code
 160 copyAccessAndModificationTime(StringRef Filename,
 161                               const sys::fs::file_status &Stat) {
 162   int FD;
 163
 164   if (auto EC =
 165           sys::fs::openFileForWrite(Filename, FD, sys::fs::CD_OpenExisting))
 166     return EC;
 167
 168   if (auto EC = sys::fs::setLastAccessAndModificationTime(
 169           FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
 170     return EC;
 171
 172   if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
 173     return EC;
 174
 175   return {};
 176 }
 177
 178 std::error_code FileCollector::copyFiles(bool StopOnError) {
 179   auto Err = sys::fs::create_directories(Root, /*IgnoreExisting=*/true);
 180   if (Err) {
 181     return Err;
 182   }
 183
 184   std::lock_guard<std::mutex> lock(Mutex);
 185
 186   for (auto &entry : VFSWriter.getMappings()) {
 187     // Get the status of the original file/directory.
 188     sys::fs::file_status Stat;
 189     if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
 190       if (StopOnError)
 191         return EC;
 192       continue;
 193     }
 194
 195     // Continue if the file doesn't exist.
 196     if (Stat.type() == sys::fs::file_type::file_not_found)
 197       continue;
 198
 199     // Create directory tree.
 200     if (std::error_code EC =
 201             sys::fs::create_directories(sys::path::parent_path(entry.RPath),
 202                                         /*IgnoreExisting=*/true)) {
 203       if (StopOnError)
 204         return EC;
 205     }
 206
 207     if (Stat.type() == sys::fs::file_type::directory_file) {
 208       // Construct a directory when it's just a directory entry.
 209       if (std::error_code EC =
 210               sys::fs::create_directories(entry.RPath,
 211                                           /*IgnoreExisting=*/true)) {
 212         if (StopOnError)
 213           return EC;
 214       }
 215       continue;
 216     }
 217
 218     // Copy file over.
 219     if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
 220       if (StopOnError)
 221         return EC;
 222     }
 223
 224     // Copy over permissions.
 225     if (auto perms = sys::fs::getPermissions(entry.VPath)) {
 226       if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
 227         if (StopOnError)
 228           return EC;
 229       }
 230     }
 231
 232     // Copy over modification time.
 233     copyAccessAndModificationTime(entry.RPath, Stat);
 234   }
 235   return {};
 236 }
 237
 238 std::error_code FileCollector::writeMapping(StringRef MappingFile) {
 239   std::lock_guard<std::mutex> lock(Mutex);
 240
 241   VFSWriter.setOverlayDir(OverlayRoot);
 242   VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
 243   VFSWriter.setUseExternalNames(false);
 244
 245   std::error_code EC;
 246   raw_fd_ostream os(MappingFile, EC, sys::fs::OF_TextWithCRLF);
 247   if (EC)
 248     return EC;
 249
 250   VFSWriter.write(os);
 251
 252   return {};
 253 }
 254
 255 namespace llvm {
 256
 257 class FileCollectorFileSystem : public vfs::FileSystem {
 258 public:
 259   explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
 260                                    std::shared_ptr<FileCollector> Collector)
 261       : FS(std::move(FS)), Collector(std::move(Collector)) {}
 262
 263   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
 264     auto Result = FS->status(Path);
 265     if (Result && Result->exists())
 266       Collector->addFile(Path);
 267     return Result;
 268   }
 269
 270   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
 271   openFileForRead(const Twine &Path) override {
 272     auto Result = FS->openFileForRead(Path);
 273     if (Result && *Result)
 274       Collector->addFile(Path);
 275     return Result;
 276   }
 277
 278   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
 279                                           std::error_code &EC) override {
 280     return Collector->addDirectoryImpl(Dir, FS, EC);
 281   }
 282
 283   std::error_code getRealPath(const Twine &Path,
 284                               SmallVectorImpl<char> &Output) const override {
 285     auto EC = FS->getRealPath(Path, Output);
 286     if (!EC) {
 287       Collector->addFile(Path);
 288       if (Output.size() > 0)
 289         Collector->addFile(Output);
 290     }
 291     return EC;
 292   }
 293
 294   std::error_code isLocal(const Twine &Path, bool &Result) override {
 295     return FS->isLocal(Path, Result);
 296   }
 297
 298   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
 299     return FS->getCurrentWorkingDirectory();
 300   }
 301
 302   std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
 303     return FS->setCurrentWorkingDirectory(Path);
 304   }
 305
 306 private:
 307   IntrusiveRefCntPtr<vfs::FileSystem> FS;
 308   std::shared_ptr<FileCollector> Collector;
 309 };
 310
 311 } // namespace llvm
 312
 313 IntrusiveRefCntPtr<vfs::FileSystem>
 314 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
 315                                   std::shared_ptr<FileCollector> Collector) {
 316   return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
 317 }