lib/Support/CachePruning.cpp

   1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file implements the pruning of a directory based on least recently used.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Support/CachePruning.h"
  15
  16 #include "llvm/Support/Debug.h"
  17 #include "llvm/Support/Errc.h"
  18 #include "llvm/Support/Error.h"
  19 #include "llvm/Support/FileSystem.h"
  20 #include "llvm/Support/Path.h"
  21 #include "llvm/Support/raw_ostream.h"
  22
  23 #define DEBUG_TYPE "cache-pruning"
  24
  25 #include <set>
  26 #include <system_error>
  27
  28 using namespace llvm;
  29
  30 namespace {
  31 struct FileInfo {
  32   sys::TimePoint<> Time;
  33   uint64_t Size;
  34   std::string Path;
  35
  36   /// Used to determine which files to prune first. Also used to determine
  37   /// set membership, so must take into account all fields.
  38   bool operator<(const FileInfo &Other) const {
  39     if (Time < Other.Time)
  40       return true;
  41     else if (Other.Time < Time)
  42       return false;
  43     if (Other.Size < Size)
  44       return true;
  45     else if (Size < Other.Size)
  46       return false;
  47     return Path < Other.Path;
  48   }
  49 };
  50 } // anonymous namespace
  51
  52 /// Write a new timestamp file with the given path. This is used for the pruning
  53 /// interval option.
  54 static void writeTimestampFile(StringRef TimestampFile) {
  55   std::error_code EC;
  56   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
  57 }
  58
  59 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
  60   if (Duration.empty())
  61     return make_error<StringError>("Duration must not be empty",
  62                                    inconvertibleErrorCode());
  63
  64   StringRef NumStr = Duration.slice(0, Duration.size()-1);
  65   uint64_t Num;
  66   if (NumStr.getAsInteger(0, Num))
  67     return make_error<StringError>("'" + NumStr + "' not an integer",
  68                                    inconvertibleErrorCode());
  69
  70   switch (Duration.back()) {
  71   case 's':
  72     return std::chrono::seconds(Num);
  73   case 'm':
  74     return std::chrono::minutes(Num);
  75   case 'h':
  76     return std::chrono::hours(Num);
  77   default:
  78     return make_error<StringError>("'" + Duration +
  79                                        "' must end with one of 's', 'm' or 'h'",
  80                                    inconvertibleErrorCode());
  81   }
  82 }
  83
  84 Expected<CachePruningPolicy>
  85 llvm::parseCachePruningPolicy(StringRef PolicyStr) {
  86   CachePruningPolicy Policy;
  87   std::pair<StringRef, StringRef> P = {"", PolicyStr};
  88   while (!P.second.empty()) {
  89     P = P.second.split(':');
  90
  91     StringRef Key, Value;
  92     std::tie(Key, Value) = P.first.split('=');
  93     if (Key == "prune_interval") {
  94       auto DurationOrErr = parseDuration(Value);
  95       if (!DurationOrErr)
  96         return DurationOrErr.takeError();
  97       Policy.Interval = *DurationOrErr;
  98     } else if (Key == "prune_after") {
  99       auto DurationOrErr = parseDuration(Value);
 100       if (!DurationOrErr)
 101         return DurationOrErr.takeError();
 102       Policy.Expiration = *DurationOrErr;
 103     } else if (Key == "cache_size") {
 104       if (Value.back() != '%')
 105         return make_error<StringError>("'" + Value + "' must be a percentage",
 106                                        inconvertibleErrorCode());
 107       StringRef SizeStr = Value.drop_back();
 108       uint64_t Size;
 109       if (SizeStr.getAsInteger(0, Size))
 110         return make_error<StringError>("'" + SizeStr + "' not an integer",
 111                                        inconvertibleErrorCode());
 112       if (Size > 100)
 113         return make_error<StringError>("'" + SizeStr +
 114                                            "' must be between 0 and 100",
 115                                        inconvertibleErrorCode());
 116       Policy.MaxSizePercentageOfAvailableSpace = Size;
 117     } else if (Key == "cache_size_bytes") {
 118       uint64_t Mult = 1;
 119       switch (tolower(Value.back())) {
 120       case 'k':
 121         Mult = 1024;
 122         Value = Value.drop_back();
 123         break;
 124       case 'm':
 125         Mult = 1024 * 1024;
 126         Value = Value.drop_back();
 127         break;
 128       case 'g':
 129         Mult = 1024 * 1024 * 1024;
 130         Value = Value.drop_back();
 131         break;
 132       }
 133       uint64_t Size;
 134       if (Value.getAsInteger(0, Size))
 135         return make_error<StringError>("'" + Value + "' not an integer",
 136                                        inconvertibleErrorCode());
 137       Policy.MaxSizeBytes = Size * Mult;
 138     } else if (Key == "cache_size_files") {
 139       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
 140         return make_error<StringError>("'" + Value + "' not an integer",
 141                                        inconvertibleErrorCode());
 142     } else {
 143       return make_error<StringError>("Unknown key: '" + Key + "'",
 144                                      inconvertibleErrorCode());
 145     }
 146   }
 147
 148   return Policy;
 149 }
 150
 151 /// Prune the cache of files that haven't been accessed in a long time.
 152 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
 153   using namespace std::chrono;
 154
 155   if (Path.empty())
 156     return false;
 157
 158   bool isPathDir;
 159   if (sys::fs::is_directory(Path, isPathDir))
 160     return false;
 161
 162   if (!isPathDir)
 163     return false;
 164
 165   Policy.MaxSizePercentageOfAvailableSpace =
 166       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
 167
 168   if (Policy.Expiration == seconds(0) &&
 169       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
 170       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
 171     LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
 172     // Nothing will be pruned, early exit
 173     return false;
 174   }
 175
 176   // Try to stat() the timestamp file.
 177   SmallString<128> TimestampFile(Path);
 178   sys::path::append(TimestampFile, "llvmcache.timestamp");
 179   sys::fs::file_status FileStatus;
 180   const auto CurrentTime = system_clock::now();
 181   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
 182     if (EC == errc::no_such_file_or_directory) {
 183       // If the timestamp file wasn't there, create one now.
 184       writeTimestampFile(TimestampFile);
 185     } else {
 186       // Unknown error?
 187       return false;
 188     }
 189   } else {
 190     if (!Policy.Interval)
 191       return false;
 192     if (Policy.Interval != seconds(0)) {
 193       // Check whether the time stamp is older than our pruning interval.
 194       // If not, do nothing.
 195       const auto TimeStampModTime = FileStatus.getLastModificationTime();
 196       auto TimeStampAge = CurrentTime - TimeStampModTime;
 197       if (TimeStampAge <= *Policy.Interval) {
 198         LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
 199                           << duration_cast<seconds>(TimeStampAge).count()
 200                           << "s old), do not prune.\n");
 201         return false;
 202       }
 203     }
 204     // Write a new timestamp file so that nobody else attempts to prune.
 205     // There is a benign race condition here, if two processes happen to
 206     // notice at the same time that the timestamp is out-of-date.
 207     writeTimestampFile(TimestampFile);
 208   }
 209
 210   // Keep track of files to delete to get below the size limit.
 211   // Order by time of last use so that recently used files are preserved.
 212   std::set<FileInfo> FileInfos;
 213   uint64_t TotalSize = 0;
 214
 215   // Walk the entire directory cache, looking for unused files.
 216   std::error_code EC;
 217   SmallString<128> CachePathNative;
 218   sys::path::native(Path, CachePathNative);
 219   // Walk all of the files within this directory.
 220   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
 221        File != FileEnd && !EC; File.increment(EC)) {
 222     // Ignore any files not beginning with the string "llvmcache-". This
 223     // includes the timestamp file as well as any files created by the user.
 224     // This acts as a safeguard against data loss if the user specifies the
 225     // wrong directory as their cache directory.
 226     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
 227       continue;
 228
 229     // Look at this file. If we can't stat it, there's nothing interesting
 230     // there.
 231     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
 232     if (!StatusOrErr) {
 233       LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
 234       continue;
 235     }
 236
 237     // If the file hasn't been used recently enough, delete it
 238     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
 239     auto FileAge = CurrentTime - FileAccessTime;
 240     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
 241       LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
 242                         << duration_cast<seconds>(FileAge).count()
 243                         << "s old)\n");
 244       sys::fs::remove(File->path());
 245       continue;
 246     }
 247
 248     // Leave it here for now, but add it to the list of size-based pruning.
 249     TotalSize += StatusOrErr->getSize();
 250     FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
 251   }
 252
 253   auto FileInfo = FileInfos.begin();
 254   size_t NumFiles = FileInfos.size();
 255
 256   auto RemoveCacheFile = [&]() {
 257     // Remove the file.
 258     sys::fs::remove(FileInfo->Path);
 259     // Update size
 260     TotalSize -= FileInfo->Size;
 261     NumFiles--;
 262     LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
 263                       << FileInfo->Size << "), new occupancy is " << TotalSize
 264                       << "%\n");
 265     ++FileInfo;
 266   };
 267
 268   // Prune for number of files.
 269   if (Policy.MaxSizeFiles)
 270     while (NumFiles > Policy.MaxSizeFiles)
 271       RemoveCacheFile();
 272
 273   // Prune for size now if needed
 274   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
 275     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
 276     if (!ErrOrSpaceInfo) {
 277       report_fatal_error("Can't get available size");
 278     }
 279     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
 280     auto AvailableSpace = TotalSize + SpaceInfo.free;
 281
 282     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
 283       Policy.MaxSizePercentageOfAvailableSpace = 100;
 284     if (Policy.MaxSizeBytes == 0)
 285       Policy.MaxSizeBytes = AvailableSpace;
 286     auto TotalSizeTarget = std::min<uint64_t>(
 287         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
 288         Policy.MaxSizeBytes);
 289
 290     LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
 291                       << "% target is: "
 292                       << Policy.MaxSizePercentageOfAvailableSpace << "%, "
 293                       << Policy.MaxSizeBytes << " bytes\n");
 294
 295     // Remove the oldest accessed files first, till we get below the threshold.
 296     while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
 297       RemoveCacheFile();
 298   }
 299   return true;
 300 }