1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the pruning of a directory based on least recently used.
11 //===----------------------------------------------------------------------===//
13 #include "llvm/Support/CachePruning.h"
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Error.h"
18 #include "llvm/Support/FileSystem.h"
19 #include "llvm/Support/Path.h"
20 #include "llvm/Support/raw_ostream.h"
22 #define DEBUG_TYPE "cache-pruning"
25 #include <system_error>
31 sys::TimePoint
<> Time
;
35 /// Used to determine which files to prune first. Also used to determine
36 /// set membership, so must take into account all fields.
37 bool operator<(const FileInfo
&Other
) const {
38 return std::tie(Time
, Other
.Size
, Path
) <
39 std::tie(Other
.Time
, Size
, Other
.Path
);
42 } // anonymous namespace
44 /// Write a new timestamp file with the given path. This is used for the pruning
46 static void writeTimestampFile(StringRef TimestampFile
) {
48 raw_fd_ostream
Out(TimestampFile
.str(), EC
, sys::fs::F_None
);
51 static Expected
<std::chrono::seconds
> parseDuration(StringRef Duration
) {
53 return make_error
<StringError
>("Duration must not be empty",
54 inconvertibleErrorCode());
56 StringRef NumStr
= Duration
.slice(0, Duration
.size()-1);
58 if (NumStr
.getAsInteger(0, Num
))
59 return make_error
<StringError
>("'" + NumStr
+ "' not an integer",
60 inconvertibleErrorCode());
62 switch (Duration
.back()) {
64 return std::chrono::seconds(Num
);
66 return std::chrono::minutes(Num
);
68 return std::chrono::hours(Num
);
70 return make_error
<StringError
>("'" + Duration
+
71 "' must end with one of 's', 'm' or 'h'",
72 inconvertibleErrorCode());
76 Expected
<CachePruningPolicy
>
77 llvm::parseCachePruningPolicy(StringRef PolicyStr
) {
78 CachePruningPolicy Policy
;
79 std::pair
<StringRef
, StringRef
> P
= {"", PolicyStr
};
80 while (!P
.second
.empty()) {
81 P
= P
.second
.split(':');
84 std::tie(Key
, Value
) = P
.first
.split('=');
85 if (Key
== "prune_interval") {
86 auto DurationOrErr
= parseDuration(Value
);
88 return DurationOrErr
.takeError();
89 Policy
.Interval
= *DurationOrErr
;
90 } else if (Key
== "prune_after") {
91 auto DurationOrErr
= parseDuration(Value
);
93 return DurationOrErr
.takeError();
94 Policy
.Expiration
= *DurationOrErr
;
95 } else if (Key
== "cache_size") {
96 if (Value
.back() != '%')
97 return make_error
<StringError
>("'" + Value
+ "' must be a percentage",
98 inconvertibleErrorCode());
99 StringRef SizeStr
= Value
.drop_back();
101 if (SizeStr
.getAsInteger(0, Size
))
102 return make_error
<StringError
>("'" + SizeStr
+ "' not an integer",
103 inconvertibleErrorCode());
105 return make_error
<StringError
>("'" + SizeStr
+
106 "' must be between 0 and 100",
107 inconvertibleErrorCode());
108 Policy
.MaxSizePercentageOfAvailableSpace
= Size
;
109 } else if (Key
== "cache_size_bytes") {
111 switch (tolower(Value
.back())) {
114 Value
= Value
.drop_back();
118 Value
= Value
.drop_back();
121 Mult
= 1024 * 1024 * 1024;
122 Value
= Value
.drop_back();
126 if (Value
.getAsInteger(0, Size
))
127 return make_error
<StringError
>("'" + Value
+ "' not an integer",
128 inconvertibleErrorCode());
129 Policy
.MaxSizeBytes
= Size
* Mult
;
130 } else if (Key
== "cache_size_files") {
131 if (Value
.getAsInteger(0, Policy
.MaxSizeFiles
))
132 return make_error
<StringError
>("'" + Value
+ "' not an integer",
133 inconvertibleErrorCode());
135 return make_error
<StringError
>("Unknown key: '" + Key
+ "'",
136 inconvertibleErrorCode());
143 /// Prune the cache of files that haven't been accessed in a long time.
144 bool llvm::pruneCache(StringRef Path
, CachePruningPolicy Policy
) {
145 using namespace std::chrono
;
151 if (sys::fs::is_directory(Path
, isPathDir
))
157 Policy
.MaxSizePercentageOfAvailableSpace
=
158 std::min(Policy
.MaxSizePercentageOfAvailableSpace
, 100u);
160 if (Policy
.Expiration
== seconds(0) &&
161 Policy
.MaxSizePercentageOfAvailableSpace
== 0 &&
162 Policy
.MaxSizeBytes
== 0 && Policy
.MaxSizeFiles
== 0) {
163 LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
164 // Nothing will be pruned, early exit
168 // Try to stat() the timestamp file.
169 SmallString
<128> TimestampFile(Path
);
170 sys::path::append(TimestampFile
, "llvmcache.timestamp");
171 sys::fs::file_status FileStatus
;
172 const auto CurrentTime
= system_clock::now();
173 if (auto EC
= sys::fs::status(TimestampFile
, FileStatus
)) {
174 if (EC
== errc::no_such_file_or_directory
) {
175 // If the timestamp file wasn't there, create one now.
176 writeTimestampFile(TimestampFile
);
182 if (!Policy
.Interval
)
184 if (Policy
.Interval
!= seconds(0)) {
185 // Check whether the time stamp is older than our pruning interval.
186 // If not, do nothing.
187 const auto TimeStampModTime
= FileStatus
.getLastModificationTime();
188 auto TimeStampAge
= CurrentTime
- TimeStampModTime
;
189 if (TimeStampAge
<= *Policy
.Interval
) {
190 LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
191 << duration_cast
<seconds
>(TimeStampAge
).count()
192 << "s old), do not prune.\n");
196 // Write a new timestamp file so that nobody else attempts to prune.
197 // There is a benign race condition here, if two processes happen to
198 // notice at the same time that the timestamp is out-of-date.
199 writeTimestampFile(TimestampFile
);
202 // Keep track of files to delete to get below the size limit.
203 // Order by time of last use so that recently used files are preserved.
204 std::set
<FileInfo
> FileInfos
;
205 uint64_t TotalSize
= 0;
207 // Walk the entire directory cache, looking for unused files.
209 SmallString
<128> CachePathNative
;
210 sys::path::native(Path
, CachePathNative
);
211 // Walk all of the files within this directory.
212 for (sys::fs::directory_iterator
File(CachePathNative
, EC
), FileEnd
;
213 File
!= FileEnd
&& !EC
; File
.increment(EC
)) {
214 // Ignore any files not beginning with the string "llvmcache-". This
215 // includes the timestamp file as well as any files created by the user.
216 // This acts as a safeguard against data loss if the user specifies the
217 // wrong directory as their cache directory.
218 if (!sys::path::filename(File
->path()).startswith("llvmcache-"))
221 // Look at this file. If we can't stat it, there's nothing interesting
223 ErrorOr
<sys::fs::basic_file_status
> StatusOrErr
= File
->status();
225 LLVM_DEBUG(dbgs() << "Ignore " << File
->path() << " (can't stat)\n");
229 // If the file hasn't been used recently enough, delete it
230 const auto FileAccessTime
= StatusOrErr
->getLastAccessedTime();
231 auto FileAge
= CurrentTime
- FileAccessTime
;
232 if (Policy
.Expiration
!= seconds(0) && FileAge
> Policy
.Expiration
) {
233 LLVM_DEBUG(dbgs() << "Remove " << File
->path() << " ("
234 << duration_cast
<seconds
>(FileAge
).count()
236 sys::fs::remove(File
->path());
240 // Leave it here for now, but add it to the list of size-based pruning.
241 TotalSize
+= StatusOrErr
->getSize();
242 FileInfos
.insert({FileAccessTime
, StatusOrErr
->getSize(), File
->path()});
245 auto FileInfo
= FileInfos
.begin();
246 size_t NumFiles
= FileInfos
.size();
248 auto RemoveCacheFile
= [&]() {
250 sys::fs::remove(FileInfo
->Path
);
252 TotalSize
-= FileInfo
->Size
;
254 LLVM_DEBUG(dbgs() << " - Remove " << FileInfo
->Path
<< " (size "
255 << FileInfo
->Size
<< "), new occupancy is " << TotalSize
260 // Prune for number of files.
261 if (Policy
.MaxSizeFiles
)
262 while (NumFiles
> Policy
.MaxSizeFiles
)
265 // Prune for size now if needed
266 if (Policy
.MaxSizePercentageOfAvailableSpace
> 0 || Policy
.MaxSizeBytes
> 0) {
267 auto ErrOrSpaceInfo
= sys::fs::disk_space(Path
);
268 if (!ErrOrSpaceInfo
) {
269 report_fatal_error("Can't get available size");
271 sys::fs::space_info SpaceInfo
= ErrOrSpaceInfo
.get();
272 auto AvailableSpace
= TotalSize
+ SpaceInfo
.free
;
274 if (Policy
.MaxSizePercentageOfAvailableSpace
== 0)
275 Policy
.MaxSizePercentageOfAvailableSpace
= 100;
276 if (Policy
.MaxSizeBytes
== 0)
277 Policy
.MaxSizeBytes
= AvailableSpace
;
278 auto TotalSizeTarget
= std::min
<uint64_t>(
279 AvailableSpace
* Policy
.MaxSizePercentageOfAvailableSpace
/ 100ull,
280 Policy
.MaxSizeBytes
);
282 LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize
) / AvailableSpace
)
284 << Policy
.MaxSizePercentageOfAvailableSpace
<< "%, "
285 << Policy
.MaxSizeBytes
<< " bytes\n");
287 // Remove the oldest accessed files first, till we get below the threshold.
288 while (TotalSize
> TotalSizeTarget
&& FileInfo
!= FileInfos
.end())