lib/Support/FileCollector.cpp

   1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Support/FileCollector.h"
  10 #include "llvm/ADT/SmallString.h"
  11 #include "llvm/Support/FileSystem.h"
  12 #include "llvm/Support/Path.h"
  13 #include "llvm/Support/Process.h"
  14
  15 using namespace llvm;
  16
  17 static bool isCaseSensitivePath(StringRef Path) {
  18   SmallString<256> TmpDest = Path, UpperDest, RealDest;
  19
  20   // Remove component traversals, links, etc.
  21   if (!sys::fs::real_path(Path, TmpDest))
  22     return true; // Current default value in vfs.yaml
  23   Path = TmpDest;
  24
  25   // Change path to all upper case and ask for its real path, if the latter
  26   // exists and is equal to path, it's not case sensitive. Default to case
  27   // sensitive in the absence of real_path, since this is the YAMLVFSWriter
  28   // default.
  29   UpperDest = Path.upper();
  30   if (sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
  31     return false;
  32   return true;
  33 }
  34
  35 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
  36     : Root(std::move(Root)), OverlayRoot(std::move(OverlayRoot)) {
  37   sys::fs::create_directories(this->Root, true);
  38 }
  39
  40 bool FileCollector::getRealPath(StringRef SrcPath,
  41                                 SmallVectorImpl<char> &Result) {
  42   SmallString<256> RealPath;
  43   StringRef FileName = sys::path::filename(SrcPath);
  44   std::string Directory = sys::path::parent_path(SrcPath).str();
  45   auto DirWithSymlink = SymlinkMap.find(Directory);
  46
  47   // Use real_path to fix any symbolic link component present in a path.
  48   // Computing the real path is expensive, cache the search through the parent
  49   // path Directory.
  50   if (DirWithSymlink == SymlinkMap.end()) {
  51     auto EC = sys::fs::real_path(Directory, RealPath);
  52     if (EC)
  53       return false;
  54     SymlinkMap[Directory] = RealPath.str();
  55   } else {
  56     RealPath = DirWithSymlink->second;
  57   }
  58
  59   sys::path::append(RealPath, FileName);
  60   Result.swap(RealPath);
  61   return true;
  62 }
  63
  64 void FileCollector::addFile(const Twine &file) {
  65   std::lock_guard<std::mutex> lock(Mutex);
  66   std::string FileStr = file.str();
  67   if (markAsSeen(FileStr))
  68     addFileImpl(FileStr);
  69 }
  70
  71 void FileCollector::addFileImpl(StringRef SrcPath) {
  72   // We need an absolute src path to append to the root.
  73   SmallString<256> AbsoluteSrc = SrcPath;
  74   sys::fs::make_absolute(AbsoluteSrc);
  75
  76   // Canonicalize src to a native path to avoid mixed separator styles.
  77   sys::path::native(AbsoluteSrc);
  78
  79   // Remove redundant leading "./" pieces and consecutive separators.
  80   AbsoluteSrc = sys::path::remove_leading_dotslash(AbsoluteSrc);
  81
  82   // Canonicalize the source path by removing "..", "." components.
  83   SmallString<256> VirtualPath = AbsoluteSrc;
  84   sys::path::remove_dots(VirtualPath, /*remove_dot_dot=*/true);
  85
  86   // If a ".." component is present after a symlink component, remove_dots may
  87   // lead to the wrong real destination path. Let the source be canonicalized
  88   // like that but make sure we always use the real path for the destination.
  89   SmallString<256> CopyFrom;
  90   if (!getRealPath(AbsoluteSrc, CopyFrom))
  91     CopyFrom = VirtualPath;
  92
  93   SmallString<256> DstPath = StringRef(Root);
  94   sys::path::append(DstPath, sys::path::relative_path(CopyFrom));
  95
  96   // Always map a canonical src path to its real path into the YAML, by doing
  97   // this we map different virtual src paths to the same entry in the VFS
  98   // overlay, which is a way to emulate symlink inside the VFS; this is also
  99   // needed for correctness, not doing that can lead to module redefinition
 100   // errors.
 101   addFileToMapping(VirtualPath, DstPath);
 102 }
 103
 104 /// Set the access and modification time for the given file from the given
 105 /// status object.
 106 static std::error_code
 107 copyAccessAndModificationTime(StringRef Filename,
 108                               const sys::fs::file_status &Stat) {
 109   int FD;
 110
 111   if (auto EC =
 112           sys::fs::openFileForWrite(Filename, FD, sys::fs::CD_OpenExisting))
 113     return EC;
 114
 115   if (auto EC = sys::fs::setLastAccessAndModificationTime(
 116           FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
 117     return EC;
 118
 119   if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
 120     return EC;
 121
 122   return {};
 123 }
 124
 125 std::error_code FileCollector::copyFiles(bool StopOnError) {
 126   for (auto &entry : VFSWriter.getMappings()) {
 127     // Create directory tree.
 128     if (std::error_code EC =
 129             sys::fs::create_directories(sys::path::parent_path(entry.RPath),
 130                                         /*IgnoreExisting=*/true)) {
 131       if (StopOnError)
 132         return EC;
 133     }
 134
 135     // Get the status of the original file/directory.
 136     sys::fs::file_status Stat;
 137     if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
 138       if (StopOnError)
 139         return EC;
 140       continue;
 141     }
 142
 143     if (Stat.type() == sys::fs::file_type::directory_file) {
 144       // Construct a directory when it's just a directory entry.
 145       if (std::error_code EC =
 146               sys::fs::create_directories(entry.RPath,
 147                                           /*IgnoreExisting=*/true)) {
 148         if (StopOnError)
 149           return EC;
 150       }
 151       continue;
 152     }
 153
 154     // Copy file over.
 155     if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
 156       if (StopOnError)
 157         return EC;
 158     }
 159
 160     // Copy over permissions.
 161     if (auto perms = sys::fs::getPermissions(entry.VPath)) {
 162       if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
 163         if (StopOnError)
 164           return EC;
 165       }
 166     }
 167
 168     // Copy over modification time.
 169     copyAccessAndModificationTime(entry.RPath, Stat);
 170   }
 171   return {};
 172 }
 173
 174 std::error_code FileCollector::writeMapping(StringRef mapping_file) {
 175   std::lock_guard<std::mutex> lock(Mutex);
 176
 177   VFSWriter.setOverlayDir(OverlayRoot);
 178   VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
 179   VFSWriter.setUseExternalNames(false);
 180
 181   std::error_code EC;
 182   raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
 183   if (EC)
 184     return EC;
 185
 186   VFSWriter.write(os);
 187
 188   return {};
 189 }
 190
 191 namespace {
 192
 193 class FileCollectorFileSystem : public vfs::FileSystem {
 194 public:
 195   explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
 196                                    std::shared_ptr<FileCollector> Collector)
 197       : FS(std::move(FS)), Collector(std::move(Collector)) {}
 198
 199   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
 200     auto Result = FS->status(Path);
 201     if (Result && Result->exists())
 202       Collector->addFile(Path);
 203     return Result;
 204   }
 205
 206   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
 207   openFileForRead(const Twine &Path) override {
 208     auto Result = FS->openFileForRead(Path);
 209     if (Result && *Result)
 210       Collector->addFile(Path);
 211     return Result;
 212   }
 213
 214   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
 215                                           std::error_code &EC) override {
 216     auto It = FS->dir_begin(Dir, EC);
 217     if (EC)
 218       return It;
 219     // Collect everything that's listed in case the user needs it.
 220     Collector->addFile(Dir);
 221     for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
 222       if (It->type() == sys::fs::file_type::regular_file ||
 223           It->type() == sys::fs::file_type::directory_file ||
 224           It->type() == sys::fs::file_type::symlink_file) {
 225         Collector->addFile(It->path());
 226       }
 227     }
 228     if (EC)
 229       return It;
 230     // Return a new iterator.
 231     return FS->dir_begin(Dir, EC);
 232   }
 233
 234   std::error_code getRealPath(const Twine &Path,
 235                               SmallVectorImpl<char> &Output) const override {
 236     auto EC = FS->getRealPath(Path, Output);
 237     if (!EC) {
 238       Collector->addFile(Path);
 239       if (Output.size() > 0)
 240         Collector->addFile(Output);
 241     }
 242     return EC;
 243   }
 244
 245   std::error_code isLocal(const Twine &Path, bool &Result) override {
 246     return FS->isLocal(Path, Result);
 247   }
 248
 249   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
 250     return FS->getCurrentWorkingDirectory();
 251   }
 252
 253   std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
 254     return FS->setCurrentWorkingDirectory(Path);
 255   }
 256
 257 private:
 258   IntrusiveRefCntPtr<vfs::FileSystem> FS;
 259   std::shared_ptr<FileCollector> Collector;
 260 };
 261
 262 } // end anonymous namespace
 263
 264 IntrusiveRefCntPtr<vfs::FileSystem>
 265 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
 266                                   std::shared_ptr<FileCollector> Collector) {
 267   return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
 268 }