lib/Support/CachePruning.cpp

   1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the pruning of a directory based on least recently used.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/Support/CachePruning.h"
  14
  15 #include "llvm/Support/Debug.h"
  16 #include "llvm/Support/Errc.h"
  17 #include "llvm/Support/Error.h"
  18 #include "llvm/Support/FileSystem.h"
  19 #include "llvm/Support/Path.h"
  20 #include "llvm/Support/raw_ostream.h"
  21
  22 #define DEBUG_TYPE "cache-pruning"
  23
  24 #include <set>
  25 #include <system_error>
  26
  27 using namespace llvm;
  28
  29 namespace {
  30 struct FileInfo {
  31   sys::TimePoint<> Time;
  32   uint64_t Size;
  33   std::string Path;
  34
  35   /// Used to determine which files to prune first. Also used to determine
  36   /// set membership, so must take into account all fields.
  37   bool operator<(const FileInfo &Other) const {
  38     if (Time < Other.Time)
  39       return true;
  40     else if (Other.Time < Time)
  41       return false;
  42     if (Other.Size < Size)
  43       return true;
  44     else if (Size < Other.Size)
  45       return false;
  46     return Path < Other.Path;
  47   }
  48 };
  49 } // anonymous namespace
  50
  51 /// Write a new timestamp file with the given path. This is used for the pruning
  52 /// interval option.
  53 static void writeTimestampFile(StringRef TimestampFile) {
  54   std::error_code EC;
  55   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
  56 }
  57
  58 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
  59   if (Duration.empty())
  60     return make_error<StringError>("Duration must not be empty",
  61                                    inconvertibleErrorCode());
  62
  63   StringRef NumStr = Duration.slice(0, Duration.size()-1);
  64   uint64_t Num;
  65   if (NumStr.getAsInteger(0, Num))
  66     return make_error<StringError>("'" + NumStr + "' not an integer",
  67                                    inconvertibleErrorCode());
  68
  69   switch (Duration.back()) {
  70   case 's':
  71     return std::chrono::seconds(Num);
  72   case 'm':
  73     return std::chrono::minutes(Num);
  74   case 'h':
  75     return std::chrono::hours(Num);
  76   default:
  77     return make_error<StringError>("'" + Duration +
  78                                        "' must end with one of 's', 'm' or 'h'",
  79                                    inconvertibleErrorCode());
  80   }
  81 }
  82
  83 Expected<CachePruningPolicy>
  84 llvm::parseCachePruningPolicy(StringRef PolicyStr) {
  85   CachePruningPolicy Policy;
  86   std::pair<StringRef, StringRef> P = {"", PolicyStr};
  87   while (!P.second.empty()) {
  88     P = P.second.split(':');
  89
  90     StringRef Key, Value;
  91     std::tie(Key, Value) = P.first.split('=');
  92     if (Key == "prune_interval") {
  93       auto DurationOrErr = parseDuration(Value);
  94       if (!DurationOrErr)
  95         return DurationOrErr.takeError();
  96       Policy.Interval = *DurationOrErr;
  97     } else if (Key == "prune_after") {
  98       auto DurationOrErr = parseDuration(Value);
  99       if (!DurationOrErr)
 100         return DurationOrErr.takeError();
 101       Policy.Expiration = *DurationOrErr;
 102     } else if (Key == "cache_size") {
 103       if (Value.back() != '%')
 104         return make_error<StringError>("'" + Value + "' must be a percentage",
 105                                        inconvertibleErrorCode());
 106       StringRef SizeStr = Value.drop_back();
 107       uint64_t Size;
 108       if (SizeStr.getAsInteger(0, Size))
 109         return make_error<StringError>("'" + SizeStr + "' not an integer",
 110                                        inconvertibleErrorCode());
 111       if (Size > 100)
 112         return make_error<StringError>("'" + SizeStr +
 113                                            "' must be between 0 and 100",
 114                                        inconvertibleErrorCode());
 115       Policy.MaxSizePercentageOfAvailableSpace = Size;
 116     } else if (Key == "cache_size_bytes") {
 117       uint64_t Mult = 1;
 118       switch (tolower(Value.back())) {
 119       case 'k':
 120         Mult = 1024;
 121         Value = Value.drop_back();
 122         break;
 123       case 'm':
 124         Mult = 1024 * 1024;
 125         Value = Value.drop_back();
 126         break;
 127       case 'g':
 128         Mult = 1024 * 1024 * 1024;
 129         Value = Value.drop_back();
 130         break;
 131       }
 132       uint64_t Size;
 133       if (Value.getAsInteger(0, Size))
 134         return make_error<StringError>("'" + Value + "' not an integer",
 135                                        inconvertibleErrorCode());
 136       Policy.MaxSizeBytes = Size * Mult;
 137     } else if (Key == "cache_size_files") {
 138       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
 139         return make_error<StringError>("'" + Value + "' not an integer",
 140                                        inconvertibleErrorCode());
 141     } else {
 142       return make_error<StringError>("Unknown key: '" + Key + "'",
 143                                      inconvertibleErrorCode());
 144     }
 145   }
 146
 147   return Policy;
 148 }
 149
 150 /// Prune the cache of files that haven't been accessed in a long time.
 151 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
 152   using namespace std::chrono;
 153
 154   if (Path.empty())
 155     return false;
 156
 157   bool isPathDir;
 158   if (sys::fs::is_directory(Path, isPathDir))
 159     return false;
 160
 161   if (!isPathDir)
 162     return false;
 163
 164   Policy.MaxSizePercentageOfAvailableSpace =
 165       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
 166
 167   if (Policy.Expiration == seconds(0) &&
 168       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
 169       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
 170     LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
 171     // Nothing will be pruned, early exit
 172     return false;
 173   }
 174
 175   // Try to stat() the timestamp file.
 176   SmallString<128> TimestampFile(Path);
 177   sys::path::append(TimestampFile, "llvmcache.timestamp");
 178   sys::fs::file_status FileStatus;
 179   const auto CurrentTime = system_clock::now();
 180   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
 181     if (EC == errc::no_such_file_or_directory) {
 182       // If the timestamp file wasn't there, create one now.
 183       writeTimestampFile(TimestampFile);
 184     } else {
 185       // Unknown error?
 186       return false;
 187     }
 188   } else {
 189     if (!Policy.Interval)
 190       return false;
 191     if (Policy.Interval != seconds(0)) {
 192       // Check whether the time stamp is older than our pruning interval.
 193       // If not, do nothing.
 194       const auto TimeStampModTime = FileStatus.getLastModificationTime();
 195       auto TimeStampAge = CurrentTime - TimeStampModTime;
 196       if (TimeStampAge <= *Policy.Interval) {
 197         LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
 198                           << duration_cast<seconds>(TimeStampAge).count()
 199                           << "s old), do not prune.\n");
 200         return false;
 201       }
 202     }
 203     // Write a new timestamp file so that nobody else attempts to prune.
 204     // There is a benign race condition here, if two processes happen to
 205     // notice at the same time that the timestamp is out-of-date.
 206     writeTimestampFile(TimestampFile);
 207   }
 208
 209   // Keep track of files to delete to get below the size limit.
 210   // Order by time of last use so that recently used files are preserved.
 211   std::set<FileInfo> FileInfos;
 212   uint64_t TotalSize = 0;
 213
 214   // Walk the entire directory cache, looking for unused files.
 215   std::error_code EC;
 216   SmallString<128> CachePathNative;
 217   sys::path::native(Path, CachePathNative);
 218   // Walk all of the files within this directory.
 219   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
 220        File != FileEnd && !EC; File.increment(EC)) {
 221     // Ignore any files not beginning with the string "llvmcache-". This
 222     // includes the timestamp file as well as any files created by the user.
 223     // This acts as a safeguard against data loss if the user specifies the
 224     // wrong directory as their cache directory.
 225     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
 226       continue;
 227
 228     // Look at this file. If we can't stat it, there's nothing interesting
 229     // there.
 230     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
 231     if (!StatusOrErr) {
 232       LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
 233       continue;
 234     }
 235
 236     // If the file hasn't been used recently enough, delete it
 237     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
 238     auto FileAge = CurrentTime - FileAccessTime;
 239     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
 240       LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
 241                         << duration_cast<seconds>(FileAge).count()
 242                         << "s old)\n");
 243       sys::fs::remove(File->path());
 244       continue;
 245     }
 246
 247     // Leave it here for now, but add it to the list of size-based pruning.
 248     TotalSize += StatusOrErr->getSize();
 249     FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
 250   }
 251
 252   auto FileInfo = FileInfos.begin();
 253   size_t NumFiles = FileInfos.size();
 254
 255   auto RemoveCacheFile = [&]() {
 256     // Remove the file.
 257     sys::fs::remove(FileInfo->Path);
 258     // Update size
 259     TotalSize -= FileInfo->Size;
 260     NumFiles--;
 261     LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
 262                       << FileInfo->Size << "), new occupancy is " << TotalSize
 263                       << "%\n");
 264     ++FileInfo;
 265   };
 266
 267   // Prune for number of files.
 268   if (Policy.MaxSizeFiles)
 269     while (NumFiles > Policy.MaxSizeFiles)
 270       RemoveCacheFile();
 271
 272   // Prune for size now if needed
 273   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
 274     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
 275     if (!ErrOrSpaceInfo) {
 276       report_fatal_error("Can't get available size");
 277     }
 278     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
 279     auto AvailableSpace = TotalSize + SpaceInfo.free;
 280
 281     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
 282       Policy.MaxSizePercentageOfAvailableSpace = 100;
 283     if (Policy.MaxSizeBytes == 0)
 284       Policy.MaxSizeBytes = AvailableSpace;
 285     auto TotalSizeTarget = std::min<uint64_t>(
 286         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
 287         Policy.MaxSizeBytes);
 288
 289     LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
 290                       << "% target is: "
 291                       << Policy.MaxSizePercentageOfAvailableSpace << "%, "
 292                       << Policy.MaxSizeBytes << " bytes\n");
 293
 294     // Remove the oldest accessed files first, till we get below the threshold.
 295     while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
 296       RemoveCacheFile();
 297   }
 298   return true;
 299 }